confluent-sql 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- confluent_sql-0.7.0/.github/CODEOWNERS +1 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/CHANGELOG.md +14 -2
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/DBAPI_EXTENSIONS.md +86 -1
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/PKG-INFO +2 -1
- confluent_sql-0.7.0/examples/artifact_udf/pyproject.toml +12 -0
- confluent_sql-0.7.0/examples/artifact_udf/src/example_udf/__init__.py +2 -0
- confluent_sql-0.7.0/examples/artifact_udf/src/example_udf/py.typed +0 -0
- confluent_sql-0.7.0/examples/artifact_udf/src/example_udf/tshirt_sizing.py +54 -0
- confluent_sql-0.7.0/examples/artifact_upload_example.py +107 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/pyproject.toml +4 -1
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/__init__.py +18 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/__version__.py +1 -1
- confluent_sql-0.7.0/src/confluent_sql/artifacts.py +711 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/connection.py +294 -97
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/connectors.py +13 -12
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/cursor.py +12 -2
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/exceptions.py +33 -0
- confluent_sql-0.7.0/src/confluent_sql/service_api.py +33 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/statement.py +10 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/utils.py +27 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/integration/conftest.py +14 -11
- confluent_sql-0.7.0/tests/integration/test_artifact.py +236 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/integration/test_connection.py +64 -1
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/integration/test_connector.py +2 -2
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/integration/test_cursor.py +1 -1
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/integration/test_fetch.py +4 -3
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/integration/test_tableflow.py +3 -2
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/conftest.py +10 -0
- confluent_sql-0.7.0/tests/unit/test_artifacts_connection_unit.py +231 -0
- confluent_sql-0.7.0/tests/unit/test_artifacts_unit.py +690 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_connection_byoidc_unit.py +2 -0
- confluent_sql-0.7.0/tests/unit/test_connection_close_unit.py +144 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_connection_connector_unit.py +1 -21
- confluent_sql-0.7.0/tests/unit/test_connection_credentials_unit.py +55 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_connection_unit.py +173 -3
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_connectors_connection_unit.py +18 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_cursor_unit.py +22 -1
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_statement_unit.py +43 -1
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_tableflow_connection_unit.py +3 -24
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_types_unit.py +3 -1
- confluent_sql-0.7.0/uv.lock +510 -0
- confluent_sql-0.6.0/.github/CODEOWNERS +0 -2
- confluent_sql-0.6.0/uv.lock +0 -466
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/.claude/rules/software-engineering.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/.gitignore +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/.semaphore/publish_to_pypi.yml +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/.semaphore/semaphore.yml +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/ARCHITECTURE.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/CLAUDE.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/LICENSE.txt +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/Makefile +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/README.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/STREAMING.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/TYPES.md +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/examples/byoidc_bearer_token_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/examples/errors.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/examples/simple_append_only_streaming_query_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/examples/snapshot_mode_tuple_cursor_simple_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/examples/statement_properties_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/examples/tableflow_gcs_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/examples/tableflow_lifecycle_example.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/service.yml +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/auth.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/changelog_compressor.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/execution_mode.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/polling.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/result_readers.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/retry.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/statement_properties.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/tableflow.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/types.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/src/confluent_sql/variant.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/__init__.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/conftest.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_auth_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_changelog_compressor_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_changelog_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_connection_unit_properties.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_connectors_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_execution_mode_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_package_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_polling_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_result_readers_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_retry_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_statement_properties_unit.py +0 -0
- {confluent_sql-0.6.0 → confluent_sql-0.7.0}/tests/unit/test_tableflow_unit.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
* @confluentinc/dbt-product @confluentinc/sql-clients
|
|
@@ -2,9 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this dbapi driver will be documented in this file.
|
|
4
4
|
|
|
5
|
-
##
|
|
5
|
+
## 0.7.0, 2026-10-19
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- `Connection.dry_run_statement(statement_text, *, mode=ExecutionMode.SNAPSHOT, compute_pool_id=None)`: validate a statement with Flink's `sql.dry-run` in a single request, without running or storing it, and return the resulting `Statement`. The returned `Statement`'s `schema` and `is_append_only` may then be of interest to the caller. Raises `OperationalError` with the server's detail if Flink rejects the statement. (#246)
|
|
10
|
+
- New property `Statement.is_dry_run`: whether the statement was submitted with the `sql.dry-run` statement property. (#246)
|
|
11
|
+
- Support for the Flink Artifact API (`/artifact/v1/flink-artifacts` and `/presigned-upload-url`), for uploading UDF packages (a JAR, or for Python a ZIP) to Confluent Cloud.
|
|
12
|
+
See `examples/artifact_upload_example.py` for a full example of packaging, uploading and calling a Python UDF artifact end to end.
|
|
13
|
+
Artifact calls authenticate with a global key, or a dedicated Cloud resource management key via the new `cloud_api_key`/`cloud_api_secret` parameters of `connect()`.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- `Cursor.execute()` rejects `sql.dry-run` property being supplied as True with an `InterfaceError` pointing to use `Connection.dry_run_statement()` instead. Previously, valid SQL raised `StatementNotFoundError` (the cursor polled for a statement the server never stores) and invalid SQL raised `OperationalError`. (#246)
|
|
18
|
+
|
|
19
|
+
## 0.6.0, 2026-09-29
|
|
8
20
|
|
|
9
21
|
### Changed - Breaking
|
|
10
22
|
|
|
@@ -33,8 +33,9 @@ For comprehensive details on streaming queries, polling patterns, and changelog
|
|
|
33
33
|
|
|
34
34
|
- [Result Format Extensions](#result-format-extensions) - Dictionary rows, custom types
|
|
35
35
|
- [Streaming Query Support](#streaming-query-support) - Comprehensive streaming guide
|
|
36
|
-
- [Statement Lifecycle Management](#statement-lifecycle-management) - DDL, naming, stopping, deletion
|
|
36
|
+
- [Statement Lifecycle Management](#statement-lifecycle-management) - DDL, dry-run, naming, stopping, deletion
|
|
37
37
|
- [Tableflow Lifecycle](#tableflow-lifecycle) - Enable, read, and disable Iceberg/Delta sinks
|
|
38
|
+
- [Flink Artifacts](#flink-artifacts) - Upload and manage UDF packages
|
|
38
39
|
- [Introspection and Metadata](#introspection-and-metadata) - Properties for query state
|
|
39
40
|
- [Performance Monitoring](#performance-monitoring) - Fetch metrics
|
|
40
41
|
- [Type System Extensions](#type-system-extensions) - Flink type support
|
|
@@ -404,6 +405,36 @@ For more details on managing named and labeled statements, see the [Statement Na
|
|
|
404
405
|
|
|
405
406
|
---
|
|
406
407
|
|
|
408
|
+
### Dry-Run Validation (`dry_run_statement()`)
|
|
409
|
+
|
|
410
|
+
Validate a statement without running it, using Flink's `sql.dry-run`. Flink validates and plans
|
|
411
|
+
the statement and answers in the submission response, without storing it, so this is a single
|
|
412
|
+
request with nothing to clean up:
|
|
413
|
+
|
|
414
|
+
```python
|
|
415
|
+
statement = connection.dry_run_statement(
|
|
416
|
+
"SELECT id, price FROM orders", mode=ExecutionMode.STREAMING_QUERY
|
|
417
|
+
)
|
|
418
|
+
print(statement.schema) # the query's result columns; None for DDL
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
- `mode` (default `ExecutionMode.SNAPSHOT`) sets `sql.snapshot.mode` as a cursor in that mode
|
|
422
|
+
would. The result schema is the same in either mode, but the statement's traits are not:
|
|
423
|
+
`is_bounded` and `is_append_only` describe a run in the mode you validated in. To learn whether
|
|
424
|
+
a streaming submission would be append-only (e.g. an unbounded aggregation is not), pass
|
|
425
|
+
`ExecutionMode.STREAMING_QUERY`; under the default, a snapshot's bounded result always reports
|
|
426
|
+
append-only.
|
|
427
|
+
- Invalid SQL raises `OperationalError` with the server's detail.
|
|
428
|
+
- The returned `Statement` has `is_dry_run` true and `statement_id` `""`. It was never stored,
|
|
429
|
+
so `get_statement()` for it raises `StatementNotFoundError`.
|
|
430
|
+
- The statement text is sent as-is (no parameter interpolation).
|
|
431
|
+
|
|
432
|
+
`Cursor.execute()` does not accept property `sql.dry-run`: a dry-run has no rows to fetch and no
|
|
433
|
+
server-side statement to manage, so it doesn't fit the cursor lifecycle. Passing it as true raises
|
|
434
|
+
`InterfaceError`, pointing to use this `Connection`-level method instead.
|
|
435
|
+
|
|
436
|
+
---
|
|
437
|
+
|
|
407
438
|
### Statement Naming and Labeling
|
|
408
439
|
|
|
409
440
|
Control statement identity and grouping for tracking and management.
|
|
@@ -741,6 +772,59 @@ for table in ("orders", "shipments", "returns"):
|
|
|
741
772
|
|
|
742
773
|
---
|
|
743
774
|
|
|
775
|
+
## Flink Artifacts
|
|
776
|
+
|
|
777
|
+
Flink artifacts are how user code -- a UDF packaged as a JAR (Java) or a ZIP (Python) -- gets into
|
|
778
|
+
Confluent Cloud Flink. Upload one, then register a function from it with
|
|
779
|
+
`CREATE FUNCTION ... USING JAR 'confluent-artifact://<artifact id>'`. Artifacts are scoped to the
|
|
780
|
+
connection's cloud, region and environment (from `connect()`'s `cloud_provider`/`cloud_region`, or a
|
|
781
|
+
standard `https://flink.<region>.<cloud>.confluent.cloud` `endpoint`).
|
|
782
|
+
|
|
783
|
+
A runnable end-to-end example -- build a Python UDF, upload it, call it from SQL, clean up -- is in
|
|
784
|
+
[examples/artifact_upload_example.py](examples/artifact_upload_example.py).
|
|
785
|
+
|
|
786
|
+
> **Credentials.** Artifacts need a global API key, or a "Cloud resource management" key passed as
|
|
787
|
+
> `cloud_api_key`/`cloud_api_secret` (used only when no global key is given). A Tableflow or Flink
|
|
788
|
+
> key won't work, and neither will BYOIDC -- like Tableflow, artifacts are a control-plane surface.
|
|
789
|
+
|
|
790
|
+
The artifact operations live on an `ArtifactsApi` object, returned by
|
|
791
|
+
`Connection.get_artifacts_api()` (composed on first call, then cached):
|
|
792
|
+
|
|
793
|
+
```python
|
|
794
|
+
from confluent_sql import ArtifactRuntimeLanguage
|
|
795
|
+
|
|
796
|
+
artifacts = connection.get_artifacts_api()
|
|
797
|
+
|
|
798
|
+
artifact = artifacts.create(
|
|
799
|
+
"my-udf",
|
|
800
|
+
"dist/my_udf-0.1.0.zip", # path of the archive to upload
|
|
801
|
+
runtime_language=ArtifactRuntimeLanguage.PYTHON,
|
|
802
|
+
description="T-shirt size comparisons",
|
|
803
|
+
)
|
|
804
|
+
artifacts.get(artifact.id).versions # versions appear on a read, not on create/list
|
|
805
|
+
artifacts.list(runtime_language="PYTHON")
|
|
806
|
+
artifacts.delete(artifact.id) # blocks until gone; wait_for_removal=False to opt out
|
|
807
|
+
```
|
|
808
|
+
|
|
809
|
+
`create` requests a presigned URL, uploads the file to it and registers the artifact. It infers
|
|
810
|
+
`content_format` from the `.jar`/`.zip` extension when you don't pass one. Only JAVA with JAR and PYTHON
|
|
811
|
+
with ZIP are accepted; anything else raises `InterfaceError` before the upload. `runtime_language`
|
|
812
|
+
defaults to JAVA, so a ZIP needs `runtime_language=PYTHON`. The file is streamed
|
|
813
|
+
from disk.
|
|
814
|
+
|
|
815
|
+
Uploads retry failures to *connect* (nothing sent yet) three times by default, but not failures
|
|
816
|
+
mid-transfer, which would re-send the whole archive. Tune with an `UploadRetry` via
|
|
817
|
+
`create(upload_retry=...)`: `UploadRetry(connect_retries=3, transfer_retries=0)`.
|
|
818
|
+
|
|
819
|
+
The Cloud API doesn't currently support adding versions to an existing artifact: each artifact has
|
|
820
|
+
a single server-assigned version, and its `display_name` is immutable. To ship new code, create a
|
|
821
|
+
new artifact rather than updating.
|
|
822
|
+
|
|
823
|
+
Errors: `ArtifactNotFoundError` (404 on get/delete), `ArtifactAlreadyExistsError` (display
|
|
824
|
+
name taken). Python UDFs run only in streaming mode.
|
|
825
|
+
|
|
826
|
+
---
|
|
827
|
+
|
|
744
828
|
## Introspection and Metadata
|
|
745
829
|
|
|
746
830
|
### Connection Properties
|
|
@@ -820,6 +904,7 @@ The `cursor.statement` property provides detailed metadata about the executed qu
|
|
|
820
904
|
| `is_append_only` | `bool` | Query produces only inserts (vs changelog with updates/deletes) |
|
|
821
905
|
| `is_bounded` | `bool` | Query has finite result set (snapshot) vs unbounded (streaming) |
|
|
822
906
|
| `is_deletable` | `bool` | Statement can be deleted |
|
|
907
|
+
| `is_dry_run` | `bool` | Statement was submitted with `sql.dry-run` (validated, never stored) |
|
|
823
908
|
| `schema` | `Schema` | Result schema with column names and types |
|
|
824
909
|
| `sql_kind` | `str` | Query type: `SELECT`, `INSERT`, `CREATE`, etc. |
|
|
825
910
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: confluent-sql
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: DB-API v2 compliant driver for Confluent Cloud Flink SQL
|
|
5
5
|
Project-URL: Repository, https://github.com/confluentinc/confluent-sql
|
|
6
6
|
Project-URL: Documentation, https://github.com/confluentinc/confluent-sql?tab=readme-ov-file#confluent-sql
|
|
@@ -227,6 +227,7 @@ Requires-Dist: httpx<0.29.0,>=0.23.0
|
|
|
227
227
|
Provides-Extra: dev
|
|
228
228
|
Requires-Dist: pytest; extra == 'dev'
|
|
229
229
|
Requires-Dist: pytest-cov; extra == 'dev'
|
|
230
|
+
Requires-Dist: pytest-xdist>=3.5.0; extra == 'dev'
|
|
230
231
|
Description-Content-Type: text/markdown
|
|
231
232
|
|
|
232
233
|
# confluent-sql
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Minimal project used by examples/artifact_upload_example.py: `uv build --sdist` of this
|
|
2
|
+
# directory produces the tarball the example zips and uploads as a Python UDF artifact.
|
|
3
|
+
[project]
|
|
4
|
+
name = "example-udf"
|
|
5
|
+
version = "0.1.0"
|
|
6
|
+
description = "Example Python UDF packaged for Confluent Cloud Flink"
|
|
7
|
+
requires-python = "~=3.11.0"
|
|
8
|
+
dependencies = ["apache-flink==2.0.0", "setuptools<81"]
|
|
9
|
+
|
|
10
|
+
[build-system]
|
|
11
|
+
requires = ["uv_build>=0.11.30,<0.12.0"]
|
|
12
|
+
build-backend = "uv_build"
|
|
File without changes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# pyright: reportMissingImports=false
|
|
2
|
+
# (pyflink is only available in the Confluent Cloud UDF runtime, not this project's environment)
|
|
3
|
+
from pyflink.table import DataTypes
|
|
4
|
+
from pyflink.table.types import DataType
|
|
5
|
+
from pyflink.table.udf import udf
|
|
6
|
+
|
|
7
|
+
# Dictionary mapping both full names and abbreviations to numeric size values
|
|
8
|
+
_SIZE_MAP = {
|
|
9
|
+
"x-small": 0,
|
|
10
|
+
"xs": 0,
|
|
11
|
+
"small": 1,
|
|
12
|
+
"s": 1,
|
|
13
|
+
"medium": 2,
|
|
14
|
+
"m": 2,
|
|
15
|
+
"large": 3,
|
|
16
|
+
"l": 3,
|
|
17
|
+
"x-large": 4,
|
|
18
|
+
"xl": 4,
|
|
19
|
+
"xx-large": 5,
|
|
20
|
+
"xxl": 5,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _get_size_value(shirt: str) -> int:
|
|
25
|
+
"""
|
|
26
|
+
Returns the numeric size value for a given shirt size string.
|
|
27
|
+
Returns -1 if the size is not found.
|
|
28
|
+
"""
|
|
29
|
+
if shirt is None:
|
|
30
|
+
return -1
|
|
31
|
+
return _SIZE_MAP.get(shirt.strip().lower(), -1)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _f_is_smaller(shirt1: str, shirt2: str) -> bool:
|
|
35
|
+
"""
|
|
36
|
+
Returns True if shirt1 is a smaller size than shirt2 based on standard T-shirt sizes.
|
|
37
|
+
If a size cannot be found, returns False.
|
|
38
|
+
"""
|
|
39
|
+
size1 = _get_size_value(shirt1)
|
|
40
|
+
size2 = _get_size_value(shirt2)
|
|
41
|
+
if size1 == -1 or size2 == -1:
|
|
42
|
+
return False
|
|
43
|
+
return size1 < size2
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
_is_smaller_inp_types: list[DataType] = [
|
|
47
|
+
DataTypes.STRING(),
|
|
48
|
+
DataTypes.STRING(),
|
|
49
|
+
]
|
|
50
|
+
is_smaller = udf(
|
|
51
|
+
_f_is_smaller,
|
|
52
|
+
input_types=_is_smaller_inp_types,
|
|
53
|
+
result_type=DataTypes.BOOLEAN(),
|
|
54
|
+
)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Example of the Flink artifact lifecycle: package a Python UDF, upload it as an artifact,
|
|
2
|
+
register it with `CREATE FUNCTION`, call it from SQL, and clean everything up.
|
|
3
|
+
|
|
4
|
+
Artifacts are how user code (a Java JAR or, as here, a Python ZIP) gets into Confluent Cloud
|
|
5
|
+
Flink. The UDF's source lives in examples/artifact_udf/ (a copy of the one built in Confluent's
|
|
6
|
+
"create a UDF" guide); this script builds its sdist with `uv build`, zips it as the API expects,
|
|
7
|
+
uploads it, and exercises it.
|
|
8
|
+
|
|
9
|
+
Artifacts are a control-plane surface and are not reachable under BYOIDC -- use an API-key
|
|
10
|
+
connection. They are scoped to the connection's cloud/region/environment.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
import subprocess
|
|
15
|
+
import tempfile
|
|
16
|
+
import zipfile
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import confluent_sql
|
|
20
|
+
from confluent_sql import ArtifactRuntimeLanguage
|
|
21
|
+
|
|
22
|
+
UDF_PROJECT = Path(__file__).parent / "artifact_udf"
|
|
23
|
+
UDF_NAME = "example_udf_is_smaller"
|
|
24
|
+
ARTIFACT_NAME = "confluent_sql_artifact_example"
|
|
25
|
+
|
|
26
|
+
conn = confluent_sql.connect(
|
|
27
|
+
global_api_key=os.environ["CONFLUENT_GLOBAL_API_KEY"],
|
|
28
|
+
global_api_secret=os.environ["CONFLUENT_GLOBAL_API_SECRET"],
|
|
29
|
+
## API-specific keys can be used instead of global API keys:
|
|
30
|
+
# cloud_api_key=os.environ["CONFLUENT_CLOUD_API_KEY"],
|
|
31
|
+
# cloud_api_secret=os.environ["CONFLUENT_CLOUD_API_SECRET"],
|
|
32
|
+
# flink_api_key=os.environ["CONFLUENT_FLINK_API_KEY"],
|
|
33
|
+
# flink_api_secret=os.environ["CONFLUENT_FLINK_API_SECRET"],
|
|
34
|
+
environment_id=os.environ["CONFLUENT_ENV_ID"],
|
|
35
|
+
organization_id=os.environ["CONFLUENT_ORG_ID"],
|
|
36
|
+
cloud_provider=os.environ["CONFLUENT_CLOUD_PROVIDER"],
|
|
37
|
+
cloud_region=os.environ["CONFLUENT_CLOUD_REGION"],
|
|
38
|
+
compute_pool_id=os.getenv("CONFLUENT_COMPUTE_POOL_ID"), # optional; None -> default pool
|
|
39
|
+
database=os.getenv("CONFLUENT_DATABASE", os.getenv("CONFLUENT_TEST_DBNAME", "")),
|
|
40
|
+
)
|
|
41
|
+
artifacts = conn.get_artifacts_api()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build_udf_zip(workdir: Path) -> Path:
|
|
45
|
+
"""Build the UDF project's sdist and wrap it in the ZIP a Python artifact must be."""
|
|
46
|
+
subprocess.run(
|
|
47
|
+
["uv", "build", "--sdist", "--out-dir", str(workdir)], cwd=UDF_PROJECT, check=True
|
|
48
|
+
)
|
|
49
|
+
(sdist,) = workdir.glob("*.tar.gz")
|
|
50
|
+
zip_path = workdir / sdist.with_suffix("").with_suffix(".zip").name
|
|
51
|
+
with zipfile.ZipFile(zip_path, "w") as zf:
|
|
52
|
+
zf.write(sdist, sdist.name)
|
|
53
|
+
return zip_path
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
artifact = None
|
|
57
|
+
try:
|
|
58
|
+
print("building the UDF package and uploading it as an artifact...")
|
|
59
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
60
|
+
udf_zip = build_udf_zip(Path(tmp))
|
|
61
|
+
# Requests a presigned URL, uploads the file to it, then creates the artifact. The .zip
|
|
62
|
+
# extension gives content_format=ZIP; Python UDFs must say runtime_language=PYTHON.
|
|
63
|
+
artifact = artifacts.create(
|
|
64
|
+
ARTIFACT_NAME,
|
|
65
|
+
file=udf_zip,
|
|
66
|
+
runtime_language=ArtifactRuntimeLanguage.PYTHON,
|
|
67
|
+
description="Created by confluent-sql's artifact_upload_example.py",
|
|
68
|
+
)
|
|
69
|
+
print(f"created artifact {artifact.id} ({artifact.display_name})")
|
|
70
|
+
|
|
71
|
+
print("artifacts in this environment/region:")
|
|
72
|
+
for a in artifacts.list(runtime_language=ArtifactRuntimeLanguage.PYTHON):
|
|
73
|
+
print(f" {a.id} {a.display_name} {a.content_format}")
|
|
74
|
+
|
|
75
|
+
# Versions only show up on a read of the artifact (not in the create response or a list).
|
|
76
|
+
artifact = artifacts.get(artifact.id)
|
|
77
|
+
print(f"versions: {[v.version for v in artifact.versions]}")
|
|
78
|
+
|
|
79
|
+
print(f"registering function {UDF_NAME} from the artifact (can take a while)...")
|
|
80
|
+
cursor = conn.cursor()
|
|
81
|
+
cursor.execute(
|
|
82
|
+
f"CREATE FUNCTION {UDF_NAME} AS 'example_udf.tshirt_sizing.is_smaller' "
|
|
83
|
+
f"LANGUAGE PYTHON USING JAR 'confluent-artifact://{artifact.id}'"
|
|
84
|
+
)
|
|
85
|
+
print("running a streaming query that calls the UDF (Python UDF startup is slow)...")
|
|
86
|
+
try:
|
|
87
|
+
# Python UDFs only run in streaming mode. A VALUES source is bounded, so this finishes.
|
|
88
|
+
with conn.closing_streaming_cursor(as_dict=True) as streaming_cursor:
|
|
89
|
+
streaming_cursor.execute(
|
|
90
|
+
f"SELECT size_a, size_b, {UDF_NAME}(size_a, size_b) AS is_smaller "
|
|
91
|
+
"FROM (VALUES ('small', 'x-large'), ('xl', 's'), ('m', 'm')) AS t(size_a, size_b)"
|
|
92
|
+
)
|
|
93
|
+
for row in streaming_cursor:
|
|
94
|
+
assert isinstance(row, dict)
|
|
95
|
+
print(f"{UDF_NAME}({row['size_a']!r}, {row['size_b']!r}) -> {row['is_smaller']}")
|
|
96
|
+
finally:
|
|
97
|
+
print(f"dropping function {UDF_NAME}...")
|
|
98
|
+
cursor.execute(f"DROP FUNCTION {UDF_NAME}")
|
|
99
|
+
finally:
|
|
100
|
+
try:
|
|
101
|
+
# wait_for_removal=True is the default: blocks until a read 404s.
|
|
102
|
+
if artifact is not None:
|
|
103
|
+
print(f"deleting artifact {artifact.id} (waits until it's gone)...")
|
|
104
|
+
artifacts.delete(artifact.id)
|
|
105
|
+
print(f"deleted artifact {artifact.id}")
|
|
106
|
+
finally:
|
|
107
|
+
conn.close()
|
|
@@ -75,6 +75,8 @@ addopts = [
|
|
|
75
75
|
"--strict-markers",
|
|
76
76
|
"--cov=confluent_sql",
|
|
77
77
|
"--cov-report=html",
|
|
78
|
+
"-n=2",
|
|
79
|
+
"--dist=loadscope",
|
|
78
80
|
]
|
|
79
81
|
testpaths = "tests"
|
|
80
82
|
markers = [
|
|
@@ -85,7 +87,7 @@ markers = [
|
|
|
85
87
|
]
|
|
86
88
|
|
|
87
89
|
[project.optional-dependencies]
|
|
88
|
-
dev = ["pytest", "pytest-cov"]
|
|
90
|
+
dev = ["pytest", "pytest-cov", "pytest-xdist>=3.5.0"]
|
|
89
91
|
|
|
90
92
|
[dependency-groups]
|
|
91
93
|
dev = [
|
|
@@ -93,6 +95,7 @@ dev = [
|
|
|
93
95
|
"pytest>=7.0.0",
|
|
94
96
|
"pytest-cov>=4.0.0",
|
|
95
97
|
"pytest-mock>=3.15.1",
|
|
98
|
+
"pytest-xdist>=3.5.0",
|
|
96
99
|
"python-dotenv>=1.0.0",
|
|
97
100
|
"ruff>=0.14.3",
|
|
98
101
|
]
|
|
@@ -6,6 +6,14 @@ executing SQL queries against Confluent SQL services.
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from .__version__ import __version__
|
|
9
|
+
from .artifacts import (
|
|
10
|
+
ArtifactContentFormat,
|
|
11
|
+
ArtifactRuntimeLanguage,
|
|
12
|
+
ArtifactsApi,
|
|
13
|
+
FlinkArtifact,
|
|
14
|
+
FlinkArtifactVersion,
|
|
15
|
+
UploadRetry,
|
|
16
|
+
)
|
|
9
17
|
from .changelog_compressor import ChangelogCompressor
|
|
10
18
|
from .connection import Connection, connect
|
|
11
19
|
from .connectors import (
|
|
@@ -17,6 +25,8 @@ from .connectors import (
|
|
|
17
25
|
)
|
|
18
26
|
from .cursor import Cursor
|
|
19
27
|
from .exceptions import (
|
|
28
|
+
ArtifactAlreadyExistsError,
|
|
29
|
+
ArtifactNotFoundError,
|
|
20
30
|
ComputePoolExhaustedError,
|
|
21
31
|
ConnectorAlreadyExistsError,
|
|
22
32
|
ConnectorNotFoundError,
|
|
@@ -99,6 +109,14 @@ __all__ = [
|
|
|
99
109
|
"TypeMismatchError",
|
|
100
110
|
"TableflowTopicNotFoundError",
|
|
101
111
|
"TableflowTopicAlreadyExistsError",
|
|
112
|
+
"ArtifactNotFoundError",
|
|
113
|
+
"ArtifactAlreadyExistsError",
|
|
114
|
+
"ArtifactsApi",
|
|
115
|
+
"ArtifactContentFormat",
|
|
116
|
+
"ArtifactRuntimeLanguage",
|
|
117
|
+
"FlinkArtifact",
|
|
118
|
+
"FlinkArtifactVersion",
|
|
119
|
+
"UploadRetry",
|
|
102
120
|
"ConnectorNotFoundError",
|
|
103
121
|
"ConnectorAlreadyExistsError",
|
|
104
122
|
"Connector",
|