confluent-sql 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- confluent_sql-0.2.0/CHANGELOG.md +24 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/DBAPI_EXTENSIONS.md +36 -7
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/PKG-INFO +4 -3
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/README.md +2 -2
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/examples/simple_append_only_streaming_query_example.py +1 -1
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/pyproject.toml +3 -1
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/__init__.py +2 -1
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/connection.py +155 -54
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/cursor.py +15 -12
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/statement.py +17 -1
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/types.py +15 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/conftest.py +13 -8
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/integration/conftest.py +8 -8
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/integration/test_cursor.py +32 -4
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/test_connection_unit.py +220 -14
- confluent_sql-0.2.0/tests/unit/test_connection_unit_properties.py +200 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/test_types_unit.py +42 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/uv.lock +1 -1
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/.github/CODEOWNERS +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/.gitignore +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/.semaphore/publish_to_codeartifact.yml +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/.semaphore/publish_to_pypi.yml +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/.semaphore/semaphore.yml +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/ARCHITECTURE.md +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/LICENSE.txt +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/Makefile +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/STREAMING.md +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/TYPES.md +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/examples/errors.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/examples/snapshot_mode_tuple_cursor_simple_example.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/service.yml +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/__version__.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/changelog_compressor.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/exceptions.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/execution_mode.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/src/confluent_sql/result_readers.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/__init__.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/integration/test_connection.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/integration/test_fetch.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/conftest.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/test_changelog_compressor_unit.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/test_changelog_unit.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/test_cursor_unit.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/test_execution_mode_unit.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/test_result_readers_unit.py +0 -0
- {confluent_sql-0.1.2 → confluent_sql-0.2.0}/tests/unit/test_statement_unit.py +0 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Change Log
|
|
2
|
+
|
|
3
|
+
All notable changes to this dbapi driver will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## Unreleased
|
|
6
|
+
|
|
7
|
+
## 0.2.0
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
* Respelled the `connect()` parameter `dbname` to `database`. The old spelling `dbname` is deprecated and will be removed in after one release cycle.
|
|
11
|
+
* Class `SqlNone` now gracefully strips trailing `NOT NULL` constraints from type names (case-insensitively), so that `str(SqlNone("DATE NOT NULL"))` returns valid FlinkSQL `"cast (null as DATE)"`.
|
|
12
|
+
* `connect()` is now keyword-only callable.
|
|
13
|
+
* The `host` parameter for `Connection.__init__()` has been renamed to `endpoint`.
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
* New optional keyword parameter `properties: PropertiesDict | None` on `Cursor.execute()` and related methods to allow callers to provide [statement execution properties](https://docs.confluent.io/cloud/current/flink/reference/statements/set.html#table-options). Note: connection or cursor-level properties for default catalog, database, and execution mode cannot be overridden by this parameter.
|
|
17
|
+
* New optional `endpoint` parameter on `connect()` and `Connection.__init__` to allow users to specify a custom Confluent Cloud API base endpoint (e.g., for private networking, staging, etc.). Mutually exclusive with (`cloud_provider`, `cloud_region`) -- either `endpoint` or (`cloud_provider`, `cloud_region`) must be provided. This replaces the `host` parameter in `Connection.__init__()`. (#66)
|
|
18
|
+
|
|
19
|
+
### Removed
|
|
20
|
+
* The unused control-plane `api_key` and `api_secret` `connect()` parameters have been removed. The Flink regional API key params `flink_api_key` and `flink_api_secret` remain.
|
|
21
|
+
|
|
22
|
+
## 0.1.x
|
|
23
|
+
|
|
24
|
+
Early access release of the driver.
|
|
@@ -808,18 +808,39 @@ cursor.execute(
|
|
|
808
808
|
timeout: int = 3000,
|
|
809
809
|
statement_name: str | None = None,
|
|
810
810
|
statement_label: str | None = None,
|
|
811
|
+
properties: dict[str, str | int | bool] | None = None,
|
|
811
812
|
) -> None
|
|
812
813
|
```
|
|
813
814
|
|
|
814
815
|
### Parameter Reference
|
|
815
816
|
|
|
816
|
-
| Parameter | Type
|
|
817
|
-
| ----------------- |
|
|
818
|
-
| `statement_text` | `str`
|
|
819
|
-
| `parameters` | `tuple \| list \| None`
|
|
820
|
-
| `timeout` | `int`
|
|
821
|
-
| `statement_name` | `str \| None`
|
|
822
|
-
| `statement_label` | `str \| None`
|
|
817
|
+
| Parameter | Type | Default | Description |
|
|
818
|
+
| ----------------- | --------------------------------- | ---------- | ------------------------------------------------------------------ |
|
|
819
|
+
| `statement_text` | `str` | (required) | SQL statement to execute |
|
|
820
|
+
| `parameters` | `tuple \| list \| None` | None | Parameter values for parameterized statements |
|
|
821
|
+
| `timeout` | `int` | 3000 | Max seconds to wait for statement to reach RUNNING/COMPLETED phase |
|
|
822
|
+
| `statement_name` | `str \| None` | None | Custom statement identifier (defaults to UUID) |
|
|
823
|
+
| `statement_label` | `str \| None` | None | Label for grouping related statements |
|
|
824
|
+
| `properties` | `dict[str, str \| int \| bool] \| None` | None | [Statement properties](#statement-properties) to set for execution |
|
|
825
|
+
|
|
826
|
+
### Statement Properties
|
|
827
|
+
|
|
828
|
+
The `properties` parameter allows you to set [Flink SQL statement properties](https://docs.confluent.io/cloud/current/flink/reference/statements/set.html#table-options) at query execution time. These are the same properties that can be set with Flink SQL `SET` statements.
|
|
829
|
+
|
|
830
|
+
**Important Precedence Rules:**
|
|
831
|
+
- Connection-level defaults (catalog, database) are always applied
|
|
832
|
+
- Cursor execution mode settings (e.g., `sql.snapshot.mode` for snapshot queries) are always applied
|
|
833
|
+
- User-provided properties in the `properties` parameter can extend these settings but cannot override system properties
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
**Accessing Properties After Execution:**
|
|
837
|
+
The properties are stored in the Statement object and can be accessed via `statement.properties`:
|
|
838
|
+
|
|
839
|
+
```python
|
|
840
|
+
cursor.execute(query, properties={"sql.state-ttl": "100 ms"})
|
|
841
|
+
props = cursor.statement.properties
|
|
842
|
+
assert props["sql.state-ttl"] == "100 ms"
|
|
843
|
+
```
|
|
823
844
|
|
|
824
845
|
**Examples:**
|
|
825
846
|
|
|
@@ -857,6 +878,14 @@ cursor.execute(
|
|
|
857
878
|
statement_name="product-sales-hourly",
|
|
858
879
|
statement_label="analytics"
|
|
859
880
|
)
|
|
881
|
+
|
|
882
|
+
# With statement properties
|
|
883
|
+
cursor.execute(
|
|
884
|
+
"SELECT * FROM orders WHERE status = %s",
|
|
885
|
+
("pending",),
|
|
886
|
+
statement_name="pending-orders-query",
|
|
887
|
+
properties={"sql.state-ttl": "100 ms"}
|
|
888
|
+
)
|
|
860
889
|
```
|
|
861
890
|
|
|
862
891
|
---
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: confluent-sql
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: DB-API v2 compliant driver for Confluent Cloud Flink SQL
|
|
5
5
|
Project-URL: Repository, https://github.com/confluentinc/confluent-sql
|
|
6
6
|
Project-URL: Documentation, https://github.com/confluentinc/confluent-sql?tab=readme-ov-file#confluent-sql
|
|
7
7
|
Project-URL: Bug Tracker, https://github.com/confluentinc/confluent-sql/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/confluentinc/confluent-sql/blob/main/CHANGELOG.md
|
|
8
9
|
Maintainer-email: Confluent <support@confluent.io>
|
|
9
10
|
License: Copyright (c) 2026 Confluent Inc.
|
|
10
11
|
|
|
@@ -244,7 +245,7 @@ This is pre-production code mainly developed as the lower level portion of a `db
|
|
|
244
245
|
|
|
245
246
|
The behavior of snapshot-mode cursors, complying with dbapi semantics, are well stable. The streaming query extensions are more of a work in progress at this time. Feedback and suggestions are welcome!
|
|
246
247
|
|
|
247
|
-
> **⚠️ Early Access:** [Snapshot queries](https://docs.confluent.io/cloud/current/flink/concepts/snapshot-queries.html) on Confluent Cloud Flink SQL are currently in Early Access and may be subject to change. The driver defaults to snapshot mode for all queries unless streaming mode is explicitly requested.
|
|
248
|
+
> **⚠️ Early Access:** [Snapshot queries](https://docs.confluent.io/cloud/current/flink/concepts/snapshot-queries.html) on Confluent Cloud Flink SQL are currently in Early Access and may be subject to change. You will need to request access to snapshot queries for your organization from Confluent. The driver defaults to snapshot mode for all queries unless streaming mode is explicitly requested.
|
|
248
249
|
|
|
249
250
|
## Prerequisites
|
|
250
251
|
|
|
@@ -278,7 +279,7 @@ connection = confluent_sql.connect(
|
|
|
278
279
|
compute_pool_id="lfcp-789012",
|
|
279
280
|
cloud_provider="aws",
|
|
280
281
|
cloud_region="us-east-2",
|
|
281
|
-
|
|
282
|
+
database="your-database-name"
|
|
282
283
|
)
|
|
283
284
|
```
|
|
284
285
|
|
|
@@ -14,7 +14,7 @@ This is pre-production code mainly developed as the lower level portion of a `db
|
|
|
14
14
|
|
|
15
15
|
The behavior of snapshot-mode cursors, complying with dbapi semantics, are well stable. The streaming query extensions are more of a work in progress at this time. Feedback and suggestions are welcome!
|
|
16
16
|
|
|
17
|
-
> **⚠️ Early Access:** [Snapshot queries](https://docs.confluent.io/cloud/current/flink/concepts/snapshot-queries.html) on Confluent Cloud Flink SQL are currently in Early Access and may be subject to change. The driver defaults to snapshot mode for all queries unless streaming mode is explicitly requested.
|
|
17
|
+
> **⚠️ Early Access:** [Snapshot queries](https://docs.confluent.io/cloud/current/flink/concepts/snapshot-queries.html) on Confluent Cloud Flink SQL are currently in Early Access and may be subject to change. You will need to request access to snapshot queries for your organization from Confluent. The driver defaults to snapshot mode for all queries unless streaming mode is explicitly requested.
|
|
18
18
|
|
|
19
19
|
## Prerequisites
|
|
20
20
|
|
|
@@ -48,7 +48,7 @@ connection = confluent_sql.connect(
|
|
|
48
48
|
compute_pool_id="lfcp-789012",
|
|
49
49
|
cloud_provider="aws",
|
|
50
50
|
cloud_region="us-east-2",
|
|
51
|
-
|
|
51
|
+
database="your-database-name"
|
|
52
52
|
)
|
|
53
53
|
```
|
|
54
54
|
|
{confluent_sql-0.1.2 → confluent_sql-0.2.0}/examples/simple_append_only_streaming_query_example.py
RENAMED
|
@@ -13,7 +13,7 @@ conn = confluent_sql.connect(
|
|
|
13
13
|
compute_pool_id=os.getenv("CONFLUENT_COMPUTE_POOL_ID", ""),
|
|
14
14
|
cloud_provider=os.getenv("CONFLUENT_CLOUD_PROVIDER", ""),
|
|
15
15
|
cloud_region=os.getenv("CONFLUENT_CLOUD_REGION", ""),
|
|
16
|
-
|
|
16
|
+
database=os.getenv("CONFLUENT_TEST_DBNAME", "default"),
|
|
17
17
|
)
|
|
18
18
|
|
|
19
19
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "confluent-sql"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "DB-API v2 compliant driver for Confluent Cloud Flink SQL"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -30,6 +30,8 @@ classifiers = [
|
|
|
30
30
|
Repository = "https://github.com/confluentinc/confluent-sql"
|
|
31
31
|
Documentation = "https://github.com/confluentinc/confluent-sql?tab=readme-ov-file#confluent-sql"
|
|
32
32
|
"Bug Tracker" = "https://github.com/confluentinc/confluent-sql/issues"
|
|
33
|
+
Changelog = "https://github.com/confluentinc/confluent-sql/blob/main/CHANGELOG.md"
|
|
34
|
+
|
|
33
35
|
|
|
34
36
|
[tool.ruff]
|
|
35
37
|
line-length = 100
|
|
@@ -27,7 +27,7 @@ from .exceptions import (
|
|
|
27
27
|
from .execution_mode import ExecutionMode
|
|
28
28
|
from .result_readers import ChangeloggedRow
|
|
29
29
|
from .statement import Op
|
|
30
|
-
from .types import SqlNone, YearMonthInterval
|
|
30
|
+
from .types import PropertiesDict, SqlNone, YearMonthInterval
|
|
31
31
|
|
|
32
32
|
# DB-API v2 module globals
|
|
33
33
|
apilevel = "2.0"
|
|
@@ -59,6 +59,7 @@ __all__ = [
|
|
|
59
59
|
"apilevel",
|
|
60
60
|
"threadsafety",
|
|
61
61
|
"paramstyle",
|
|
62
|
+
"PropertiesDict",
|
|
62
63
|
"SqlNone",
|
|
63
64
|
"YearMonthInterval",
|
|
64
65
|
]
|
|
@@ -24,22 +24,23 @@ from .exceptions import InterfaceError, OperationalError, StatementDeletedError
|
|
|
24
24
|
from .execution_mode import ExecutionMode
|
|
25
25
|
from .statement import LABEL_PREFIX as STATEMENT_LABEL_PREFIX
|
|
26
26
|
from .statement import ChangelogRow, Statement
|
|
27
|
-
from .types import RowPythonTypes
|
|
27
|
+
from .types import PropertiesDict, RowPythonTypes
|
|
28
28
|
|
|
29
29
|
logger = logging.getLogger(__name__)
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
def connect( # noqa: PLR0913
|
|
33
|
+
*,
|
|
33
34
|
flink_api_key: str,
|
|
34
35
|
flink_api_secret: str,
|
|
35
36
|
environment: str,
|
|
36
37
|
compute_pool_id: str,
|
|
37
38
|
organization_id: str,
|
|
38
|
-
cloud_provider: str,
|
|
39
|
-
cloud_region: str,
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
dbname: str | None = None,
|
|
39
|
+
cloud_provider: str | None = None,
|
|
40
|
+
cloud_region: str | None = None,
|
|
41
|
+
database: str | None = None,
|
|
42
|
+
endpoint: str | None = None,
|
|
43
|
+
dbname: str | None = None, # deprecated, use database parameter
|
|
43
44
|
result_page_fetch_pause_millis: int = 100,
|
|
44
45
|
http_user_agent: str | None = None,
|
|
45
46
|
) -> Connection:
|
|
@@ -47,16 +48,23 @@ def connect( # noqa: PLR0913
|
|
|
47
48
|
Create a connection to a Confluent SQL service.
|
|
48
49
|
|
|
49
50
|
Args:
|
|
50
|
-
flink_api_key: Flink API key
|
|
51
|
-
flink_api_secret: Flink API secret
|
|
51
|
+
flink_api_key: Flink Region API key
|
|
52
|
+
flink_api_secret: Flink Region API secret
|
|
52
53
|
environment: Environment ID
|
|
53
54
|
compute_pool_id: Compute pool ID for SQL execution
|
|
54
55
|
organization_id: Organization ID
|
|
55
|
-
cloud_provider: Cloud provider (e.g., "aws", "gcp", "azure")
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
56
|
+
cloud_provider: Cloud provider (e.g., "aws", "gcp", "azure"). Required if endpoint is not
|
|
57
|
+
provided; must not be provided if endpoint is specified.
|
|
58
|
+
cloud_region: Cloud region (e.g., "us-east-2", "us-west-2"). Required if endpoint is not
|
|
59
|
+
provided; must not be provided if endpoint is specified.
|
|
60
|
+
database: The default Flink database (Kafka cluster) to use when resolving
|
|
61
|
+
table/view/udf names (optional)
|
|
62
|
+
endpoint: The base URL for Confluent Cloud API (optional). If not provided, the endpoint
|
|
63
|
+
will be constructed based on the cloud_provider and cloud_region parameters in the
|
|
64
|
+
format "https://flink.{cloud_region}.{cloud_provider}.confluent.cloud". A trailing
|
|
65
|
+
slash is optional and will be stripped if provided (e.g., both
|
|
66
|
+
"https://custom.example.com" and "https://custom.example.com/" are accepted).
|
|
67
|
+
dbname: Deprecated alias for database parameter (optional)
|
|
60
68
|
result_page_fetch_pause_millis: Maximum milliseconds to wait between fetching pages of
|
|
61
69
|
statement results (per statement). Defaults to 100ms. Prevents tight loops of requests
|
|
62
70
|
to the statement results API when consuming results for a statement, especially when
|
|
@@ -88,15 +96,33 @@ def connect( # noqa: PLR0913
|
|
|
88
96
|
if not organization_id:
|
|
89
97
|
raise InterfaceError("Organization ID is required")
|
|
90
98
|
|
|
91
|
-
if
|
|
92
|
-
|
|
99
|
+
if endpoint:
|
|
100
|
+
if cloud_provider or cloud_region:
|
|
101
|
+
raise InterfaceError(
|
|
102
|
+
"cloud_provider and cloud_region should not be provided when endpoint is specified"
|
|
103
|
+
)
|
|
104
|
+
else:
|
|
105
|
+
if not cloud_provider:
|
|
106
|
+
raise InterfaceError("Cloud provider is required when endpoint is not provided")
|
|
93
107
|
|
|
94
|
-
|
|
95
|
-
|
|
108
|
+
if not cloud_region:
|
|
109
|
+
raise InterfaceError("Cloud region is required when endpoint is not provided")
|
|
96
110
|
|
|
97
111
|
if not flink_api_key or not flink_api_secret:
|
|
98
112
|
raise InterfaceError("Flink API key and secret are required")
|
|
99
113
|
|
|
114
|
+
if dbname is not None:
|
|
115
|
+
warnings.warn(
|
|
116
|
+
"The 'dbname' parameter is deprecated and will be removed in a future release. "
|
|
117
|
+
"Please use the 'database' parameter instead.",
|
|
118
|
+
DeprecationWarning,
|
|
119
|
+
stacklevel=2,
|
|
120
|
+
)
|
|
121
|
+
if database is not None:
|
|
122
|
+
raise InterfaceError(
|
|
123
|
+
"Cannot specify both 'database' and deprecated 'dbname' parameters"
|
|
124
|
+
)
|
|
125
|
+
|
|
100
126
|
return Connection(
|
|
101
127
|
flink_api_key,
|
|
102
128
|
flink_api_secret,
|
|
@@ -105,9 +131,8 @@ def connect( # noqa: PLR0913
|
|
|
105
131
|
organization_id,
|
|
106
132
|
cloud_provider,
|
|
107
133
|
cloud_region,
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
dbname=dbname,
|
|
134
|
+
endpoint,
|
|
135
|
+
database=database or dbname, # dbname is deprecated.
|
|
111
136
|
statement_results_page_fetch_pause_millis=result_page_fetch_pause_millis,
|
|
112
137
|
http_user_agent=http_user_agent,
|
|
113
138
|
)
|
|
@@ -128,9 +153,6 @@ class Connection:
|
|
|
128
153
|
environment: str
|
|
129
154
|
organization_id: str
|
|
130
155
|
compute_pool_id: str
|
|
131
|
-
api_key: str | None
|
|
132
|
-
api_secret: str | None
|
|
133
|
-
host: str | None
|
|
134
156
|
statement_results_page_fetch_pause_secs: float
|
|
135
157
|
"""Maximum seconds to wait between fetching pages of statement
|
|
136
158
|
results (per statement). Prevents tight loops of requests to the
|
|
@@ -147,7 +169,7 @@ class Connection:
|
|
|
147
169
|
"""
|
|
148
170
|
|
|
149
171
|
_closed: bool
|
|
150
|
-
|
|
172
|
+
_database: str | None
|
|
151
173
|
_client: httpx.Client
|
|
152
174
|
_http_user_agent: str
|
|
153
175
|
|
|
@@ -165,12 +187,10 @@ class Connection:
|
|
|
165
187
|
environment: str,
|
|
166
188
|
compute_pool_id: str,
|
|
167
189
|
organization_id: str,
|
|
168
|
-
cloud_provider: str,
|
|
169
|
-
cloud_region: str,
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
host: str | None = None,
|
|
173
|
-
dbname: str | None = None,
|
|
190
|
+
cloud_provider: str | None,
|
|
191
|
+
cloud_region: str | None,
|
|
192
|
+
endpoint: str | None,
|
|
193
|
+
database: str | None = None,
|
|
174
194
|
statement_results_page_fetch_pause_millis: int = 100,
|
|
175
195
|
http_user_agent: str | None = None,
|
|
176
196
|
):
|
|
@@ -183,16 +203,18 @@ class Connection:
|
|
|
183
203
|
environment: Environment ID
|
|
184
204
|
compute_pool_id: Compute pool ID for SQL execution
|
|
185
205
|
organization_id: Organization ID
|
|
186
|
-
cloud_provider: Cloud provider
|
|
187
|
-
cloud_region: Cloud region (e.g., "us-east-2", "us-west-2")
|
|
206
|
+
cloud_provider: Cloud provider (required if endpoint is not provided)
|
|
207
|
+
cloud_region: Cloud region (e.g., "us-east-2", "us-west-2"). Required if endpoint is
|
|
208
|
+
not provided.
|
|
209
|
+
endpoint: The base URL for Confluent Cloud API (optional). If not provided, the
|
|
210
|
+
endpoint will be constructed based on the cloud_provider and cloud_region
|
|
211
|
+
parameters in the format "https://flink.{cloud_region}.{cloud_provider}.confluent.cloud".
|
|
212
|
+
A trailing slash is optional and will be stripped if provided.
|
|
213
|
+
database: The name of the database to use (optional)
|
|
188
214
|
result_page_fetch_pause_millis: Milliseconds to possibly wait between fetching pages of
|
|
189
215
|
statement results. Defaults to 100ms. If most recent fetch of results for a
|
|
190
216
|
statement was more than this long ago, then no delay will happen when fetching
|
|
191
217
|
the next page of results for the statement.
|
|
192
|
-
api_key: Confluent Cloud API key for general Confluent Cloud resources (optional)
|
|
193
|
-
api_secret: Confluent Cloud API secret for general Confluent Cloud resources (optional)
|
|
194
|
-
host: The base URL for Confluent Cloud API (optional)
|
|
195
|
-
dbname: The name of the database to use (optional)
|
|
196
218
|
http_user_agent: User-Agent header for HTTP requests. String, 1-100 chars.
|
|
197
219
|
Defaults to the value of DEFAULT_USER_AGENT, which includes the
|
|
198
220
|
driver name/version, documentation URL, and support email.
|
|
@@ -200,9 +222,6 @@ class Connection:
|
|
|
200
222
|
self.environment = environment
|
|
201
223
|
self.compute_pool_id = compute_pool_id
|
|
202
224
|
self.organization_id = organization_id
|
|
203
|
-
self.api_key = api_key
|
|
204
|
-
self.api_secret = api_secret
|
|
205
|
-
self.host = host
|
|
206
225
|
|
|
207
226
|
if statement_results_page_fetch_pause_millis < 0:
|
|
208
227
|
raise InterfaceError("result_page_fetch_pause_millis must be non-negative")
|
|
@@ -215,17 +234,27 @@ class Connection:
|
|
|
215
234
|
|
|
216
235
|
# Internal state
|
|
217
236
|
self._closed = False
|
|
218
|
-
self.
|
|
237
|
+
self._database = database
|
|
219
238
|
|
|
220
239
|
# Set user agent (validation happens in setter, default if None)
|
|
221
240
|
self.http_user_agent = (
|
|
222
241
|
http_user_agent if http_user_agent is not None else self.DEFAULT_USER_AGENT
|
|
223
242
|
)
|
|
224
243
|
|
|
244
|
+
if not endpoint and not (cloud_provider and cloud_region):
|
|
245
|
+
raise InterfaceError(
|
|
246
|
+
"cloud_provider and cloud_region are required when endpoint is not provided"
|
|
247
|
+
)
|
|
248
|
+
|
|
225
249
|
# Create httpx client for making API calls
|
|
226
|
-
if
|
|
227
|
-
|
|
228
|
-
|
|
250
|
+
if not endpoint:
|
|
251
|
+
# Construct the endpoint URL based on cloud provider and region.
|
|
252
|
+
endpoint = f"https://flink.{cloud_region}.{cloud_provider}.confluent.cloud"
|
|
253
|
+
else:
|
|
254
|
+
# Strip trailing slash if user provided one, to ensure clean URL construction
|
|
255
|
+
endpoint = endpoint.rstrip("/")
|
|
256
|
+
|
|
257
|
+
base_url = f"{endpoint}/sql/v1/organizations/{organization_id}/environments/{environment}"
|
|
229
258
|
|
|
230
259
|
# Create httpx client for making API calls
|
|
231
260
|
basic_auth = httpx.BasicAuth(username=flink_api_key, password=flink_api_secret)
|
|
@@ -477,6 +506,7 @@ class Connection:
|
|
|
477
506
|
timeout: int = 3000,
|
|
478
507
|
statement_name: str | None = None,
|
|
479
508
|
statement_label: str | None = None,
|
|
509
|
+
properties: PropertiesDict | None = None,
|
|
480
510
|
) -> Statement:
|
|
481
511
|
"""Execute bounded DDL that completes after consuming snapshot data.
|
|
482
512
|
|
|
@@ -499,6 +529,8 @@ class Connection:
|
|
|
499
529
|
group and manage related statements. The label will be
|
|
500
530
|
prefixed with "user.confluent.io/" when stored but you only
|
|
501
531
|
need to provide the label value itself (e.g., "my-ddl-batch")
|
|
532
|
+
properties: Optional dictionary of statement properties to set for this execution.
|
|
533
|
+
Keys must be strings, values must be str, int, or bool.
|
|
502
534
|
|
|
503
535
|
Returns:
|
|
504
536
|
Statement for managing the statement lifecycle
|
|
@@ -514,6 +546,7 @@ class Connection:
|
|
|
514
546
|
timeout=timeout,
|
|
515
547
|
statement_name=statement_name,
|
|
516
548
|
statement_label=statement_label,
|
|
549
|
+
properties=properties,
|
|
517
550
|
)
|
|
518
551
|
|
|
519
552
|
# Return the last version of the statement
|
|
@@ -526,6 +559,7 @@ class Connection:
|
|
|
526
559
|
timeout: int = 3000,
|
|
527
560
|
statement_name: str | None = None,
|
|
528
561
|
statement_label: str | None = None,
|
|
562
|
+
properties: PropertiesDict | None = None,
|
|
529
563
|
) -> Statement:
|
|
530
564
|
"""Execute unbounded DDL that starts a streaming job.
|
|
531
565
|
|
|
@@ -544,6 +578,8 @@ class Connection:
|
|
|
544
578
|
group and manage related statements. The label will be
|
|
545
579
|
prefixed with "user.confluent.io/" when stored but you only
|
|
546
580
|
need to provide the label value itself (e.g., "streaming-jobs")
|
|
581
|
+
properties: Optional dictionary of statement properties to set for this execution.
|
|
582
|
+
Keys must be strings, values must be str, int, or bool.
|
|
547
583
|
Returns:
|
|
548
584
|
Statement for any further management of the statement lifecycle
|
|
549
585
|
"""
|
|
@@ -555,6 +591,7 @@ class Connection:
|
|
|
555
591
|
timeout=timeout,
|
|
556
592
|
statement_name=statement_name,
|
|
557
593
|
statement_label=statement_label,
|
|
594
|
+
properties=properties,
|
|
558
595
|
)
|
|
559
596
|
|
|
560
597
|
return cur.statement
|
|
@@ -739,12 +776,79 @@ class Connection:
|
|
|
739
776
|
|
|
740
777
|
self._row_type_registry.register_row_type(class_for_flink_row)
|
|
741
778
|
|
|
779
|
+
_NEVER_USER_PROVIDED_PROPERTIES = {
|
|
780
|
+
"sql.current-catalog",
|
|
781
|
+
"sql.current-database",
|
|
782
|
+
"sql.snapshot.mode",
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
def _resolve_properties(
|
|
786
|
+
self, properties: PropertiesDict | None, execution_mode: ExecutionMode
|
|
787
|
+
) -> PropertiesDict:
|
|
788
|
+
"""
|
|
789
|
+
Validate and merge user properties with system properties.
|
|
790
|
+
|
|
791
|
+
Validates the properties parameter and merges it with system-level properties
|
|
792
|
+
(catalog, database, snapshot mode). System properties always have precedence
|
|
793
|
+
and cannot be overridden by user input.
|
|
794
|
+
|
|
795
|
+
Args:
|
|
796
|
+
properties: Optional dictionary of user-provided statement properties.
|
|
797
|
+
Keys must be strings, values must be str, int, or bool.
|
|
798
|
+
execution_mode: The execution mode (determines if snapshot.mode is set).
|
|
799
|
+
|
|
800
|
+
Returns:
|
|
801
|
+
Merged properties dictionary with system properties overlaid.
|
|
802
|
+
|
|
803
|
+
Raises:
|
|
804
|
+
InterfaceError: If properties parameter is invalid (not a dict, invalid keys/values).
|
|
805
|
+
"""
|
|
806
|
+
# Validate properties parameter if provided
|
|
807
|
+
if properties is not None:
|
|
808
|
+
if not isinstance(properties, dict):
|
|
809
|
+
raise InterfaceError(f"properties must be a dict, got {type(properties).__name__}")
|
|
810
|
+
|
|
811
|
+
for key, value in properties.items():
|
|
812
|
+
if not isinstance(key, str):
|
|
813
|
+
raise InterfaceError(
|
|
814
|
+
f"properties keys must be strings, got {type(key).__name__} for key {key!r}"
|
|
815
|
+
)
|
|
816
|
+
if not isinstance(value, (str, int, bool)):
|
|
817
|
+
raise InterfaceError(
|
|
818
|
+
f"properties values must be str, int, or bool, "
|
|
819
|
+
f"got {type(value).__name__} for key {key!r}"
|
|
820
|
+
)
|
|
821
|
+
if key in self._NEVER_USER_PROVIDED_PROPERTIES:
|
|
822
|
+
raise InterfaceError(f"'{key}' is a reserved system property.")
|
|
823
|
+
|
|
824
|
+
# Start with user properties (if provided), then overlay system properties
|
|
825
|
+
# This ensures system properties always win and cannot be overridden
|
|
826
|
+
merged_properties: PropertiesDict = {}
|
|
827
|
+
|
|
828
|
+
if properties is not None:
|
|
829
|
+
# User properties applied first
|
|
830
|
+
merged_properties.update(properties)
|
|
831
|
+
|
|
832
|
+
# Connection-level properties overlay (always set, cannot be overridden by user)
|
|
833
|
+
merged_properties["sql.current-catalog"] = self.environment
|
|
834
|
+
|
|
835
|
+
if self._database is not None:
|
|
836
|
+
merged_properties["sql.current-database"] = self._database
|
|
837
|
+
|
|
838
|
+
# Cursor-level execution mode properties overlay (always set when applicable)
|
|
839
|
+
if execution_mode.is_snapshot:
|
|
840
|
+
# Ask for snapshot mode behavior -- point-in-time results.
|
|
841
|
+
merged_properties["sql.snapshot.mode"] = "now"
|
|
842
|
+
|
|
843
|
+
return merged_properties
|
|
844
|
+
|
|
742
845
|
def _execute_statement(
|
|
743
846
|
self,
|
|
744
847
|
statement: str,
|
|
745
848
|
execution_mode: ExecutionMode,
|
|
746
849
|
statement_name: str | None = None,
|
|
747
850
|
statement_label: str | None = None,
|
|
851
|
+
properties: PropertiesDict | None = None,
|
|
748
852
|
) -> dict[str, Any]:
|
|
749
853
|
"""
|
|
750
854
|
Execute a SQL statement and return the response.
|
|
@@ -755,28 +859,25 @@ class Connection:
|
|
|
755
859
|
statement_name: Optional name for the statement (defaults to 'dbapi-{uuid}')
|
|
756
860
|
statement_label: Optional label for the statement for easier identification in
|
|
757
861
|
server logs and UIs (defaults to None).
|
|
862
|
+
properties: Optional dictionary of statement properties to set for this execution.
|
|
863
|
+
Keys must be strings, values must be str, int, or bool. System
|
|
864
|
+
properties (sql.current-catalog, sql.current-database, sql.snapshot.mode)
|
|
865
|
+
are always set by the driver and cannot be overridden.
|
|
758
866
|
|
|
759
867
|
Returns:
|
|
760
868
|
Dictionary containing the API response
|
|
761
869
|
|
|
762
870
|
Raises:
|
|
763
871
|
OperationalError: If statement execution fails
|
|
872
|
+
InterfaceError: If properties parameter is invalid
|
|
764
873
|
"""
|
|
765
874
|
|
|
766
875
|
# Create the statement payload as per Flink SQL API documentation
|
|
767
876
|
if statement_name is None:
|
|
768
877
|
statement_name = f"dbapi-{str(uuid.uuid4())}"
|
|
769
878
|
|
|
770
|
-
#
|
|
771
|
-
|
|
772
|
-
properties = {"sql.current-catalog": self.environment}
|
|
773
|
-
|
|
774
|
-
if self._dbname is not None:
|
|
775
|
-
properties["sql.current-database"] = self._dbname
|
|
776
|
-
|
|
777
|
-
if execution_mode.is_snapshot:
|
|
778
|
-
# Ask for snapshot mode behavior -- point-in-time results.
|
|
779
|
-
properties["sql.snapshot.mode"] = "now"
|
|
879
|
+
# Resolve and merge user properties with system properties
|
|
880
|
+
merged_properties = self._resolve_properties(properties, execution_mode)
|
|
780
881
|
|
|
781
882
|
payload = {
|
|
782
883
|
"name": statement_name,
|
|
@@ -784,7 +885,7 @@ class Connection:
|
|
|
784
885
|
"environment_id": self.environment,
|
|
785
886
|
"spec": {
|
|
786
887
|
"statement": statement,
|
|
787
|
-
"properties":
|
|
888
|
+
"properties": merged_properties,
|
|
788
889
|
"compute_pool_id": self.compute_pool_id,
|
|
789
890
|
"stopped": False,
|
|
790
891
|
},
|