confluent-sql 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/.semaphore/publish_to_pypi.yml +1 -1
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/.semaphore/semaphore.yml +4 -6
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/ARCHITECTURE.md +13 -12
- confluent_sql-0.4.0/CHANGELOG.md +62 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/DBAPI_EXTENSIONS.md +56 -3
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/PKG-INFO +18 -12
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/README.md +17 -11
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/pyproject.toml +2 -2
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/connection.py +340 -69
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/cursor.py +94 -43
- confluent_sql-0.4.0/src/confluent_sql/polling.py +57 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/statement.py +20 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/conftest.py +10 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/integration/conftest.py +59 -13
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/integration/test_connection.py +144 -2
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/integration/test_cursor.py +329 -266
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/conftest.py +27 -3
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_connection_unit.py +787 -13
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_cursor_unit.py +148 -115
- confluent_sql-0.4.0/tests/unit/test_polling_unit.py +124 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_statement_unit.py +39 -0
- confluent_sql-0.4.0/uv.lock +439 -0
- confluent_sql-0.3.0/.semaphore/publish_to_codeartifact.yml +0 -24
- confluent_sql-0.3.0/CHANGELOG.md +0 -37
- confluent_sql-0.3.0/uv.lock +0 -439
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/.github/CODEOWNERS +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/.gitignore +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/LICENSE.txt +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/Makefile +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/STREAMING.md +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/TYPES.md +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/examples/errors.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/examples/simple_append_only_streaming_query_example.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/examples/snapshot_mode_tuple_cursor_simple_example.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/service.yml +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/__init__.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/__version__.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/changelog_compressor.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/exceptions.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/execution_mode.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/result_readers.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/src/confluent_sql/types.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/__init__.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/integration/test_fetch.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_changelog_compressor_unit.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_changelog_unit.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_connection_unit_properties.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_execution_mode_unit.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_result_readers_unit.py +0 -0
- {confluent_sql-0.3.0 → confluent_sql-0.4.0}/tests/unit/test_types_unit.py +0 -0
|
@@ -21,7 +21,7 @@ global_job_config:
|
|
|
21
21
|
- checkout
|
|
22
22
|
- . vault-setup
|
|
23
23
|
- curl -LsSf https://astral.sh/uv/0.10.11/install.sh | sh
|
|
24
|
-
- uv sync
|
|
24
|
+
- uv sync --frozen
|
|
25
25
|
- export VERSION=$(grep '^version' pyproject.toml | cut -d'"' -f2)
|
|
26
26
|
|
|
27
27
|
blocks:
|
|
@@ -31,19 +31,17 @@ blocks:
|
|
|
31
31
|
jobs:
|
|
32
32
|
- name: "Lint, Type Check, and Test"
|
|
33
33
|
commands:
|
|
34
|
-
- uv run ruff check && uv run pyright && uv run pytest -m unit --junitxml=test-results.xml
|
|
34
|
+
- uv run --frozen ruff check && uv run --frozen pyright && uv run --frozen pytest -m unit --junitxml=test-results.xml
|
|
35
35
|
epilogue:
|
|
36
36
|
always:
|
|
37
37
|
commands:
|
|
38
38
|
- "[[ -f test-results.xml ]] && test-results publish test-results.xml"
|
|
39
39
|
|
|
40
40
|
promotions:
|
|
41
|
-
- name: Publish confluent-sql to codeartifact
|
|
42
|
-
pipeline_file: publish_to_codeartifact.yml
|
|
43
|
-
auto_promote:
|
|
44
|
-
when: "result = 'passed' and tag =~ '^v\\.'"
|
|
45
41
|
- name: Publish confluent-sql to PyPi
|
|
46
42
|
pipeline_file: publish_to_pypi.yml
|
|
43
|
+
auto_promote:
|
|
44
|
+
when: "result = 'passed' and tag =~ '^v\\.'"
|
|
47
45
|
|
|
48
46
|
after_pipeline:
|
|
49
47
|
task:
|
|
@@ -109,28 +109,29 @@ print(statement.name) # "create-summary-table"
|
|
|
109
109
|
|
|
110
110
|
**Name Uniqueness:**
|
|
111
111
|
|
|
112
|
-
Statement names **must be unique within the
|
|
112
|
+
Statement names **must be unique within the environment** they are submitted to. If you attempt to submit a statement with a name that already exists anywhere in the same environment—even from a connection on a different compute pool—the server will reject it with an error. This means:
|
|
113
113
|
|
|
114
|
-
- Auto-generated names are guaranteed unique
|
|
115
|
-
- If using explicit names, you must ensure they don't conflict with existing
|
|
114
|
+
- Auto-generated names are guaranteed unique within the environment
|
|
115
|
+
- If using explicit names, you must ensure they don't conflict with any existing statement in the environment
|
|
116
116
|
- After deleting a statement, its name becomes available for reuse
|
|
117
|
-
- Names are scoped to the
|
|
117
|
+
- Names are scoped to the environment, not to a compute pool—a name in use anywhere in the environment cannot be reused from another compute pool
|
|
118
118
|
|
|
119
119
|
```python
|
|
120
|
-
# This will
|
|
120
|
+
# This will fail: the name is already in use elsewhere in the environment,
|
|
121
|
+
# even though the second connection targets a different compute pool.
|
|
121
122
|
pool_a_cursor = connection_pool_a.cursor()
|
|
122
123
|
pool_a_cursor.execute(
|
|
123
124
|
"SELECT * FROM data",
|
|
124
|
-
statement_name="my-query" # OK in
|
|
125
|
+
statement_name="my-query" # OK: first use of this name in the environment
|
|
125
126
|
)
|
|
126
127
|
|
|
127
128
|
pool_b_cursor = connection_pool_b.cursor()
|
|
128
129
|
pool_b_cursor.execute(
|
|
129
130
|
"SELECT * FROM data",
|
|
130
|
-
statement_name="my-query" #
|
|
131
|
+
statement_name="my-query" # ❌ Error: name already exists in the environment
|
|
131
132
|
)
|
|
132
133
|
|
|
133
|
-
# This
|
|
134
|
+
# This also fails for the same reason, within a single compute pool:
|
|
134
135
|
cursor1 = connection.cursor()
|
|
135
136
|
cursor1.execute("SELECT * FROM table1", statement_name="my-query")
|
|
136
137
|
|
|
@@ -167,7 +168,7 @@ Labels allow you to organize statements logically without requiring unique names
|
|
|
167
168
|
|
|
168
169
|
### Statement Persistence and Recovery
|
|
169
170
|
|
|
170
|
-
Statements persist on the server independently of your client connection,
|
|
171
|
+
Statements persist on the server independently of your client connection, and are **scoped to the environment** (organization + environment) they were submitted to—not to the compute pool or the submitting connection. Any connection to the same environment can find, monitor, and recover them, regardless of which compute pool ran them (narrow to a single pool with the `compute_pool_id` filter on `list_statements()` when you want pool-level isolation):
|
|
171
172
|
|
|
172
173
|
1. **Create a statement with a name** for recovery:
|
|
173
174
|
|
|
@@ -180,9 +181,9 @@ Statements persist on the server independently of your client connection, but ar
|
|
|
180
181
|
)
|
|
181
182
|
```
|
|
182
183
|
|
|
183
|
-
2. **Recover it from another connection in the same
|
|
184
|
+
2. **Recover it from another connection in the same environment:**
|
|
184
185
|
```python
|
|
185
|
-
# In a different Python process or session (same
|
|
186
|
+
# In a different Python process or session (same environment)
|
|
186
187
|
statements = connection.list_statements(label="daily-jobs")
|
|
187
188
|
for stmt in statements:
|
|
188
189
|
if stmt.name == "daily-summary-job":
|
|
@@ -292,7 +293,7 @@ This simplifies your code by eliminating manual polling loops, but your applicat
|
|
|
292
293
|
|
|
293
294
|
### Finding Existing Statements
|
|
294
295
|
|
|
295
|
-
List statements by label (useful for batch operations):
|
|
296
|
+
List statements—optionally filtered by label, compute pool, and/or name substring (useful for batch operations):
|
|
296
297
|
|
|
297
298
|
```python
|
|
298
299
|
# Find all statements with a specific label
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Change Log
|
|
2
|
+
|
|
3
|
+
All notable changes to this dbapi driver will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## 0.4.0, 2026-06-15
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- Support for "Global" Confluent Cloud API keys: `confluent_sql.connect()` (and `Connection`) now accept `global_api_key` / `global_api_secret`. A Global key works against every route this driver touches, so it is preferred over a Flink Region key when both are supplied. `flink_api_key` / `flink_api_secret` remain supported and are now optional; at least one fully-specified pair must be provided. If both pairs are supplied, the Global pair is used (and the Flink pair ignored, with a warning); a half-specified pair (key without secret, or vice versa) is rejected. (#112)
|
|
10
|
+
- Support for "poolless Flink": `confluent_sql.connect()` now treats `compute_pool_id` as optional. Statements submitted w/o their own overriding `compute_pool_id` via a connection w/o a default compute pool id will make use of the default compute pool in the environment+cloud region (provisioning one if necessary). See [the Confluent documentation](https://docs.confluent.io/cloud/current/flink/concepts/compute-pools.html#default-compute-pools) for more details.
|
|
11
|
+
- New `Connection.stop_statement(statement, *, wait_for_stopped=True, timeout=300)` method to stop a running statement without deleting it, leaving the statement resource around for inspection (unlike `delete_statement()`, which also destroys it). Accepts a statement name or a `Statement` object. By default it blocks until the statement reaches `STOPPED`; pass `wait_for_stopped=False` to return as soon as the stop is accepted. A matching `Cursor.stop_statement()` stops the cursor's current statement. New `Statement.is_stopped`, `Statement.is_stopping`, and `Statement.stop_requested` properties expose the relevant state. (#61)
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
|
|
15
|
+
- `Connection.list_statements()`:
|
|
16
|
+
- New optional parameter `compute_pool_id` to list statements only in a single compute pool (otherwise environment-wide).
|
|
17
|
+
- New optional parameter `name_contains: str` to filter statements server-side to those whose name contains the given substring (case-sensitive).
|
|
18
|
+
- Existing parameter `label` is now optional.
|
|
19
|
+
- The end result is that callers can now provide between zero and all of the possible kwargs to vary between 'no filtering at all, return all current statements in the environment' and 'apply all the possible filters as if ANDed together.'
|
|
20
|
+
|
|
21
|
+
## 0.3.1, 2026-05-21
|
|
22
|
+
|
|
23
|
+
### Added
|
|
24
|
+
|
|
25
|
+
- New `http_timeout_secs` parameter for `connect()` to let the caller control how long to wait in HTTP requests.
|
|
26
|
+
|
|
27
|
+
## 0.3.0, 2026-04-09
|
|
28
|
+
|
|
29
|
+
### Changed - Breaking
|
|
30
|
+
|
|
31
|
+
- `connect()` / `Connection.__init__()`: Renamed `environment` parameter to `environment_id` to clarify that an environment ID (_not_ name) is expected. The internal attribute `Connection.environment` has also been renamed to `Connection.environment_id`. Update all calls from `connect(environment="env-123")` to `connect(environment_id="env-123")`. (#92)
|
|
32
|
+
- `Cursor.execute()` and peers: Respelled and re-typed the `statement_label: str | None` parameter to be `statement_labels: list[str] | None` to allow multiple labels to be applied to a statement, including `HIDDEN_LABEL`.
|
|
33
|
+
|
|
34
|
+
### Added
|
|
35
|
+
|
|
36
|
+
- New `Connection.get_statement(statement)` method to retrieve statement metadata by name or refresh a Statement object with the latest server state. Accepts either a statement name (string) or a Statement object. Returns a Statement object with current phase, schema, and execution traits. (#86)
|
|
37
|
+
- New `StatementNotFoundError` exception, a subclass of `OperationalError`, raised by `Connection.get_statement(statement)` when attempting to retrieve a statement that does not exist. Provides programmatic access to the statement name via the `statement_name` attribute.
|
|
38
|
+
- New constant `confluent_sql.HIDDEN_LABEL` used for driving `Cursor.execute()` to indicate that the statement should be hidden in default listings in Confluent Cloud UIs. This feature is intended to be used for minor queries, such as when investigating `INFORMATION_SCHEMA`.
|
|
39
|
+
- Added documentation regarding use of `connect(endpoint=)` parameter to make use of private networking endpoints (README.md, docstrings).
|
|
40
|
+
|
|
41
|
+
## 0.2.0, 2026-03-26
|
|
42
|
+
|
|
43
|
+
### Changed
|
|
44
|
+
|
|
45
|
+
- Respelled the `connect()` parameter `dbname` to `database`. The old spelling `dbname` is deprecated and will be removed in after one release cycle.
|
|
46
|
+
- Class `SqlNone` now gracefully strips trailing `NOT NULL` constraints from type names (case-insensitively), so that `str(SqlNone("DATE NOT NULL"))` returns valid FlinkSQL `"cast (null as DATE)"`.
|
|
47
|
+
- `connect()` is now keyword-only callable.
|
|
48
|
+
- The `host` parameter for `Connection.__init__()` has been renamed to `endpoint`.
|
|
49
|
+
- Clarified and improved documentation around Flink region API key use.
|
|
50
|
+
|
|
51
|
+
### Added
|
|
52
|
+
|
|
53
|
+
- New optional keyword parameter `properties: PropertiesDict | None` on `Cursor.execute()` and related methods to allow callers to provide [statement execution properties](https://docs.confluent.io/cloud/current/flink/reference/statements/set.html#table-options). Note: connection or cursor-level properties for default catalog, database, and execution mode cannot be overridden by this parameter.
|
|
54
|
+
- New optional `endpoint` parameter on `connect()` and `Connection.__init__` to allow users to specify a custom Confluent Cloud API base endpoint (e.g., for private networking, staging, etc.). Mutually exclusive with (`cloud_provider`, `cloud_region`) -- either `endpoint` or (`cloud_provider`, `cloud_region`) must be provided. This replaces the `host` parameter in `Connection.__init__()`. (#66)
|
|
55
|
+
|
|
56
|
+
### Removed
|
|
57
|
+
|
|
58
|
+
- The unused control-plane `api_key` and `api_secret` `connect()` parameters have been removed. The Flink Region API key params `flink_api_key` and `flink_api_secret` remain.
|
|
59
|
+
|
|
60
|
+
## 0.1.x
|
|
61
|
+
|
|
62
|
+
Early access release of the driver.
|
|
@@ -35,7 +35,7 @@ For comprehensive details on streaming queries, polling patterns, and changelog
|
|
|
35
35
|
|
|
36
36
|
- [Result Format Extensions](#result-format-extensions) - Dictionary rows, custom types
|
|
37
37
|
- [Streaming Query Support](#streaming-query-support) - Comprehensive streaming guide
|
|
38
|
-
- [Statement Lifecycle Management](#statement-lifecycle-management) - DDL, naming, deletion
|
|
38
|
+
- [Statement Lifecycle Management](#statement-lifecycle-management) - DDL, naming, stopping, deletion
|
|
39
39
|
- [Introspection and Metadata](#introspection-and-metadata) - Properties for query state
|
|
40
40
|
- [Performance Monitoring](#performance-monitoring) - Fetch metrics
|
|
41
41
|
- [Type System Extensions](#type-system-extensions) - Flink type support
|
|
@@ -455,11 +455,23 @@ for stmt in statements:
|
|
|
455
455
|
|
|
456
456
|
### Finding and Deleting Statements
|
|
457
457
|
|
|
458
|
-
**`list_statements()` - Find statements
|
|
458
|
+
**`list_statements()` - Find statements, optionally filtered**
|
|
459
|
+
|
|
460
|
+
Called with no arguments, returns every statement in the environment. Three optional,
|
|
461
|
+
server-side filters narrow the results and combine with AND semantics:
|
|
462
|
+
|
|
463
|
+
- `label` — statements carrying the given end-user label.
|
|
464
|
+
- `compute_pool_id` — statements in a specific compute pool.
|
|
465
|
+
- `name_contains` — statements whose name contains the given substring (case-sensitive).
|
|
459
466
|
|
|
460
467
|
```python
|
|
461
|
-
#
|
|
468
|
+
# Every statement in the environment
|
|
469
|
+
statements = connection.list_statements()
|
|
470
|
+
|
|
471
|
+
# Narrow by label, compute pool, and/or name substring (filters AND together)
|
|
462
472
|
statements = connection.list_statements(label="daily-backups", page_size=100)
|
|
473
|
+
statements = connection.list_statements(compute_pool_id="lfcp-789012")
|
|
474
|
+
statements = connection.list_statements(name_contains="orders-stream")
|
|
463
475
|
|
|
464
476
|
for statement in statements:
|
|
465
477
|
print(f"Statement: {statement.name}")
|
|
@@ -494,6 +506,47 @@ except OperationalError as e:
|
|
|
494
506
|
print(f"Other error: {e}")
|
|
495
507
|
```
|
|
496
508
|
|
|
509
|
+
**`stop_statement()` - Halt a statement without deleting it**
|
|
510
|
+
|
|
511
|
+
Stops a running statement (for example, a long-lived streaming query) while keeping the
|
|
512
|
+
statement resource around for inspection — unlike `delete_statement()`, which stops *and*
|
|
513
|
+
destroys the statement and its results. The stop is issued as a JSON Patch flipping
|
|
514
|
+
`spec.stopped` to `true`; the statement then transitions through `STOPPING` to the terminal
|
|
515
|
+
`STOPPED` phase.
|
|
516
|
+
|
|
517
|
+
```python
|
|
518
|
+
# Stop from connection (by name or Statement object), blocking until STOPPED (the default)
|
|
519
|
+
stopped = connection.stop_statement("active-users-query")
|
|
520
|
+
print(stopped.is_stopped) # True
|
|
521
|
+
|
|
522
|
+
# Non-blocking: return as soon as the stop is accepted
|
|
523
|
+
stmt = connection.stop_statement(statement_obj, wait_for_stopped=False)
|
|
524
|
+
print(stmt.stop_requested) # True (the stop was accepted)
|
|
525
|
+
# Note: stmt.phase may still be RUNNING at this instant -- the server transitions the phase
|
|
526
|
+
# to STOPPED asynchronously. Poll get_statement() if you need to observe STOPPED:
|
|
527
|
+
while not connection.get_statement(stmt).is_stopped:
|
|
528
|
+
time.sleep(0.5)
|
|
529
|
+
|
|
530
|
+
# Bound the blocking wait (seconds); raises OperationalError on timeout
|
|
531
|
+
stopped = connection.stop_statement("active-users-query", timeout=60)
|
|
532
|
+
|
|
533
|
+
# Stop from cursor (current statement); updates the cursor's tracked statement
|
|
534
|
+
stopped = cursor.stop_statement()
|
|
535
|
+
```
|
|
536
|
+
|
|
537
|
+
**Behavior notes:**
|
|
538
|
+
|
|
539
|
+
- Accepts a statement name (string) or a `Statement` object. A `Statement` already in a terminal
|
|
540
|
+
phase (`STOPPED`/`COMPLETED`/`FAILED`/`DELETED`) is returned unchanged without an API call.
|
|
541
|
+
- `wait_for_stopped=True` (default) blocks until the statement reaches a terminal phase —
|
|
542
|
+
normally `STOPPED`, but `COMPLETED` if a bounded query finished before the stop landed — so the
|
|
543
|
+
caller knows the statement is no longer running. `wait_for_stopped=False` returns once the stop
|
|
544
|
+
is accepted — confirm acceptance via `Statement.stop_requested` rather than the phase.
|
|
545
|
+
- Raises `StatementNotFoundError` if the statement does not exist, or `OperationalError` on other
|
|
546
|
+
API errors, on timeout, or if the statement transitions to `FAILED` while stopping.
|
|
547
|
+
- `cursor.stop_statement()` raises `InterfaceError` when the cursor has no executed statement to
|
|
548
|
+
stop (it is not a silent no-op like `cursor.delete_statement()`).
|
|
549
|
+
|
|
497
550
|
**`delete_statement()` - Stop and remove a statement**
|
|
498
551
|
|
|
499
552
|
```python
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: confluent-sql
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: DB-API v2 compliant driver for Confluent Cloud Flink SQL
|
|
5
5
|
Project-URL: Repository, https://github.com/confluentinc/confluent-sql
|
|
6
6
|
Project-URL: Documentation, https://github.com/confluentinc/confluent-sql?tab=readme-ov-file#confluent-sql
|
|
@@ -250,13 +250,16 @@ The behavior of snapshot-mode cursors, complying with dbapi semantics, are well
|
|
|
250
250
|
## Prerequisites
|
|
251
251
|
|
|
252
252
|
- **Confluent Cloud account** with Flink environment
|
|
253
|
-
- **Active Flink compute pool** (must be pre-created)
|
|
254
253
|
- **Existing Flink Database** (Confluent Cloud Kafka cluster)
|
|
255
|
-
- **
|
|
254
|
+
- **API credentials**, one of:
|
|
255
|
+
- a **"Global" Confluent Cloud API key** and secret, passed as `global_api_key` / `global_api_secret`, or
|
|
256
|
+
- a **Flink Region API key** and secret, passed as `flink_api_key` / `flink_api_secret`.
|
|
257
|
+
|
|
258
|
+
A Global key works against every route this driver touches, so it is the more future-proof choice; a Flink Region key works against the Flink SQL routes that are the driver's focus today. Provide at least one pair. If you supply both, the Global pair is used and the Flink pair is ignored. A half-supplied pair (a key without its secret, or vice versa) is rejected.
|
|
256
259
|
|
|
257
260
|
### How to Obtain a Flink Region API Key
|
|
258
261
|
|
|
259
|
-
|
|
262
|
+
A **Flink Region API key** (also called a Flink SQL API key) is specific to your Flink region/environment (the environment + cloud provider + cloud region triplet) and provides access to the regional Flink SQL API endpoints. This is distinct from a Confluent Cloud control-plane API key. (A "Global" Confluent Cloud API key is the alternative — see above — and is created the same way but without scoping to a single Flink region.)
|
|
260
263
|
|
|
261
264
|
To create or find a Flink Region API key:
|
|
262
265
|
|
|
@@ -266,12 +269,13 @@ To create or find a Flink Region API key:
|
|
|
266
269
|
4. When creating a new key, select either:
|
|
267
270
|
- **My account** - for development/testing
|
|
268
271
|
- **Service account** - for production applications (recommended)
|
|
269
|
-
|
|
272
|
+
and then:
|
|
270
273
|
- **The Environment, Cloud Provider and Cloud Region** matching the Flink database(s) / Kafka cluster(s) you intend to use this driver against.
|
|
271
274
|
5. Save both the **API key** and **API secret** securely (the secret cannot be retrieved later)
|
|
272
275
|
6. Use these credentials as the `flink_api_key` and `flink_api_secret` parameters in the `connect()` function
|
|
273
276
|
|
|
274
277
|
API keys may also be generated using the Confluent CLI or by API access, outside the scope of this document.
|
|
278
|
+
|
|
275
279
|
## Installation
|
|
276
280
|
|
|
277
281
|
```bash
|
|
@@ -295,10 +299,10 @@ connection = confluent_sql.connect(
|
|
|
295
299
|
environment_id="env-123456",
|
|
296
300
|
cloud_provider="aws",
|
|
297
301
|
cloud_region="us-east-2",
|
|
298
|
-
flink_api_key="your-flink-api-key",
|
|
299
|
-
flink_api_secret="your-flink-api-secret",
|
|
302
|
+
flink_api_key="your-flink-api-key", # or global_api_key=... for a "Global" Confluent Cloud key
|
|
303
|
+
flink_api_secret="your-flink-api-secret", # or global_api_secret=...
|
|
300
304
|
database="your-database-name",
|
|
301
|
-
compute_pool_id="lfcp-789012"
|
|
305
|
+
compute_pool_id="lfcp-789012" # optional; omit to use the environment's default compute pool
|
|
302
306
|
)
|
|
303
307
|
```
|
|
304
308
|
|
|
@@ -398,15 +402,15 @@ For detailed streaming query guidance, see **[STREAMING.md](https://github.com/c
|
|
|
398
402
|
|
|
399
403
|
For type support and examples, see **[TYPES.md](https://github.com/confluentinc/confluent-sql/blob/main/TYPES.md)**.
|
|
400
404
|
|
|
401
|
-
|
|
402
405
|
## Private Networking Considerations
|
|
403
406
|
|
|
404
407
|
By default, this driver uses the public Confluent Cloud API networking endpoint for the provided cloud provider and region. However, if the Flink database / Kafka cluster you intend to query requires private networking connectivity, then provide the appropriate Flink private networking base URL as the `endpoint` parameter
|
|
405
408
|
to `connect()` or `Connection.__init__()`. Refer to the Confluent Cloud [Flink private networking documentation](https://docs.confluent.io/cloud/current/flink/concepts/flink-private-networking.html) for more information on composing your endpoint URL.
|
|
406
409
|
|
|
407
410
|
Symptoms of using the public endpoint when private networking is required include:
|
|
408
|
-
|
|
409
|
-
|
|
411
|
+
|
|
412
|
+
- HTTP 429-related exceptions raised when submitting statements querying tables whose backing Kafka topics / clusters are configured for private networking only.
|
|
413
|
+
- Empty or surprisingly missing results when querying `INFORMATION_SCHEMA` or `SHOW TABLES`, due to silent filtering of private-networking-only tables/topics when querying the system catalog.
|
|
410
414
|
|
|
411
415
|
## Development
|
|
412
416
|
|
|
@@ -439,8 +443,10 @@ export CONFLUENT_CLOUD_PROVIDER="aws"
|
|
|
439
443
|
export CONFLUENT_CLOUD_REGION="us-east-2"
|
|
440
444
|
export CONFLUENT_FLINK_API_KEY="your-key" # Flink Region API key for the above cloud/region ...
|
|
441
445
|
export CONFLUENT_FLINK_API_SECRET="your-secret" # and associated secret.
|
|
442
|
-
|
|
446
|
+
# Alternatively, a "Global" Confluent Cloud API key (used in preference to the Flink pair if both
|
|
447
|
+
# are set): export CONFLUENT_GLOBAL_API_KEY / CONFLUENT_GLOBAL_API_SECRET instead.
|
|
443
448
|
export CONFLUENT_TEST_DBNAME="test-db" # A database/kafka cluster name within the above cloud/region.
|
|
449
|
+
export CONFLUENT_COMPUTE_POOL_ID="lfcp-789012" # Optional. If set, the integration suite runs against this pool; if unset, the suite runs against the environment's default pool. The driver treats it as optional at connect() either way.
|
|
444
450
|
```
|
|
445
451
|
|
|
446
452
|
Run tests:
|
|
@@ -19,13 +19,16 @@ The behavior of snapshot-mode cursors, complying with dbapi semantics, are well
|
|
|
19
19
|
## Prerequisites
|
|
20
20
|
|
|
21
21
|
- **Confluent Cloud account** with Flink environment
|
|
22
|
-
- **Active Flink compute pool** (must be pre-created)
|
|
23
22
|
- **Existing Flink Database** (Confluent Cloud Kafka cluster)
|
|
24
|
-
- **
|
|
23
|
+
- **API credentials**, one of:
|
|
24
|
+
- a **"Global" Confluent Cloud API key** and secret, passed as `global_api_key` / `global_api_secret`, or
|
|
25
|
+
- a **Flink Region API key** and secret, passed as `flink_api_key` / `flink_api_secret`.
|
|
26
|
+
|
|
27
|
+
A Global key works against every route this driver touches, so it is the more future-proof choice; a Flink Region key works against the Flink SQL routes that are the driver's focus today. Provide at least one pair. If you supply both, the Global pair is used and the Flink pair is ignored. A half-supplied pair (a key without its secret, or vice versa) is rejected.
|
|
25
28
|
|
|
26
29
|
### How to Obtain a Flink Region API Key
|
|
27
30
|
|
|
28
|
-
|
|
31
|
+
A **Flink Region API key** (also called a Flink SQL API key) is specific to your Flink region/environment (the environment + cloud provider + cloud region triplet) and provides access to the regional Flink SQL API endpoints. This is distinct from a Confluent Cloud control-plane API key. (A "Global" Confluent Cloud API key is the alternative — see above — and is created the same way but without scoping to a single Flink region.)
|
|
29
32
|
|
|
30
33
|
To create or find a Flink Region API key:
|
|
31
34
|
|
|
@@ -35,12 +38,13 @@ To create or find a Flink Region API key:
|
|
|
35
38
|
4. When creating a new key, select either:
|
|
36
39
|
- **My account** - for development/testing
|
|
37
40
|
- **Service account** - for production applications (recommended)
|
|
38
|
-
|
|
41
|
+
and then:
|
|
39
42
|
- **The Environment, Cloud Provider and Cloud Region** matching the Flink database(s) / Kafka cluster(s) you intend to use this driver against.
|
|
40
43
|
5. Save both the **API key** and **API secret** securely (the secret cannot be retrieved later)
|
|
41
44
|
6. Use these credentials as the `flink_api_key` and `flink_api_secret` parameters in the `connect()` function
|
|
42
45
|
|
|
43
46
|
API keys may also be generated using the Confluent CLI or by API access, outside the scope of this document.
|
|
47
|
+
|
|
44
48
|
## Installation
|
|
45
49
|
|
|
46
50
|
```bash
|
|
@@ -64,10 +68,10 @@ connection = confluent_sql.connect(
|
|
|
64
68
|
environment_id="env-123456",
|
|
65
69
|
cloud_provider="aws",
|
|
66
70
|
cloud_region="us-east-2",
|
|
67
|
-
flink_api_key="your-flink-api-key",
|
|
68
|
-
flink_api_secret="your-flink-api-secret",
|
|
71
|
+
flink_api_key="your-flink-api-key", # or global_api_key=... for a "Global" Confluent Cloud key
|
|
72
|
+
flink_api_secret="your-flink-api-secret", # or global_api_secret=...
|
|
69
73
|
database="your-database-name",
|
|
70
|
-
compute_pool_id="lfcp-789012"
|
|
74
|
+
compute_pool_id="lfcp-789012" # optional; omit to use the environment's default compute pool
|
|
71
75
|
)
|
|
72
76
|
```
|
|
73
77
|
|
|
@@ -167,15 +171,15 @@ For detailed streaming query guidance, see **[STREAMING.md](https://github.com/c
|
|
|
167
171
|
|
|
168
172
|
For type support and examples, see **[TYPES.md](https://github.com/confluentinc/confluent-sql/blob/main/TYPES.md)**.
|
|
169
173
|
|
|
170
|
-
|
|
171
174
|
## Private Networking Considerations
|
|
172
175
|
|
|
173
176
|
By default, this driver uses the public Confluent Cloud API networking endpoint for the provided cloud provider and region. However, if the Flink database / Kafka cluster you intend to query requires private networking connectivity, then provide the appropriate Flink private networking base URL as the `endpoint` parameter
|
|
174
177
|
to `connect()` or `Connection.__init__()`. Refer to the Confluent Cloud [Flink private networking documentation](https://docs.confluent.io/cloud/current/flink/concepts/flink-private-networking.html) for more information on composing your endpoint URL.
|
|
175
178
|
|
|
176
179
|
Symptoms of using the public endpoint when private networking is required include:
|
|
177
|
-
|
|
178
|
-
|
|
180
|
+
|
|
181
|
+
- HTTP 429-related exceptions raised when submitting statements querying tables whose backing Kafka topics / clusters are configured for private networking only.
|
|
182
|
+
- Empty or surprisingly missing results when querying `INFORMATION_SCHEMA` or `SHOW TABLES`, due to silent filtering of private-networking-only tables/topics when querying the system catalog.
|
|
179
183
|
|
|
180
184
|
## Development
|
|
181
185
|
|
|
@@ -208,8 +212,10 @@ export CONFLUENT_CLOUD_PROVIDER="aws"
|
|
|
208
212
|
export CONFLUENT_CLOUD_REGION="us-east-2"
|
|
209
213
|
export CONFLUENT_FLINK_API_KEY="your-key" # Flink Region API key for the above cloud/region ...
|
|
210
214
|
export CONFLUENT_FLINK_API_SECRET="your-secret" # and associated secret.
|
|
211
|
-
|
|
215
|
+
# Alternatively, a "Global" Confluent Cloud API key (used in preference to the Flink pair if both
|
|
216
|
+
# are set): export CONFLUENT_GLOBAL_API_KEY / CONFLUENT_GLOBAL_API_SECRET instead.
|
|
212
217
|
export CONFLUENT_TEST_DBNAME="test-db" # A database/kafka cluster name within the above cloud/region.
|
|
218
|
+
export CONFLUENT_COMPUTE_POOL_ID="lfcp-789012" # Optional. If set, the integration suite runs against this pool; if unset, the suite runs against the environment's default pool. The driver treats it as optional at connect() either way.
|
|
213
219
|
```
|
|
214
220
|
|
|
215
221
|
Run tests:
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "confluent-sql"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "DB-API v2 compliant driver for Confluent Cloud Flink SQL"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -85,7 +85,7 @@ dev = ["pytest", "pytest-cov"]
|
|
|
85
85
|
|
|
86
86
|
[dependency-groups]
|
|
87
87
|
dev = [
|
|
88
|
-
"pyright>=1.1.
|
|
88
|
+
"pyright>=1.1.410",
|
|
89
89
|
"pytest>=7.0.0",
|
|
90
90
|
"pytest-cov>=4.0.0",
|
|
91
91
|
"pytest-mock>=3.15.1",
|