confluent-sql 0.5.5__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- confluent_sql-0.7.0/.claude/rules/software-engineering.md +52 -0
- confluent_sql-0.7.0/.github/CODEOWNERS +1 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/.semaphore/publish_to_pypi.yml +2 -3
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/.semaphore/semaphore.yml +1 -1
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/ARCHITECTURE.md +26 -17
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/CHANGELOG.md +43 -1
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/DBAPI_EXTENSIONS.md +128 -9
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/PKG-INFO +2 -1
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/TYPES.md +1 -0
- confluent_sql-0.7.0/examples/artifact_udf/pyproject.toml +12 -0
- confluent_sql-0.7.0/examples/artifact_udf/src/example_udf/__init__.py +2 -0
- confluent_sql-0.7.0/examples/artifact_udf/src/example_udf/py.typed +0 -0
- confluent_sql-0.7.0/examples/artifact_udf/src/example_udf/tshirt_sizing.py +54 -0
- confluent_sql-0.7.0/examples/artifact_upload_example.py +107 -0
- confluent_sql-0.7.0/examples/tableflow_gcs_example.py +98 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/examples/tableflow_lifecycle_example.py +1 -1
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/pyproject.toml +9 -2
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/__init__.py +33 -1
- confluent_sql-0.7.0/src/confluent_sql/__version__.py +3 -0
- confluent_sql-0.7.0/src/confluent_sql/artifacts.py +711 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/connection.py +487 -201
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/connectors.py +13 -12
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/cursor.py +26 -3
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/exceptions.py +33 -0
- confluent_sql-0.7.0/src/confluent_sql/service_api.py +33 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/statement.py +125 -26
- confluent_sql-0.7.0/src/confluent_sql/tableflow.py +585 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/types.py +72 -39
- confluent_sql-0.7.0/src/confluent_sql/utils.py +62 -0
- confluent_sql-0.7.0/src/confluent_sql/variant.py +239 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/integration/conftest.py +14 -11
- confluent_sql-0.7.0/tests/integration/test_artifact.py +236 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/integration/test_connection.py +105 -4
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/integration/test_connector.py +2 -2
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/integration/test_cursor.py +133 -5
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/integration/test_fetch.py +169 -46
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/integration/test_tableflow.py +4 -3
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/conftest.py +12 -0
- confluent_sql-0.7.0/tests/unit/test_artifacts_connection_unit.py +231 -0
- confluent_sql-0.7.0/tests/unit/test_artifacts_unit.py +690 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_connection_byoidc_unit.py +2 -0
- confluent_sql-0.7.0/tests/unit/test_connection_close_unit.py +144 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_connection_connector_unit.py +1 -21
- confluent_sql-0.7.0/tests/unit/test_connection_credentials_unit.py +55 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_connection_unit.py +277 -74
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_connectors_connection_unit.py +18 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_cursor_unit.py +136 -12
- confluent_sql-0.7.0/tests/unit/test_package_unit.py +10 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_statement_unit.py +390 -18
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_tableflow_connection_unit.py +206 -33
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_tableflow_unit.py +131 -5
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_types_unit.py +223 -21
- confluent_sql-0.7.0/uv.lock +510 -0
- confluent_sql-0.5.5/.github/CODEOWNERS +0 -2
- confluent_sql-0.5.5/src/confluent_sql/__version__.py +0 -10
- confluent_sql-0.5.5/src/confluent_sql/tableflow.py +0 -368
- confluent_sql-0.5.5/uv.lock +0 -439
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/.gitignore +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/CLAUDE.md +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/LICENSE.txt +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/Makefile +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/README.md +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/STREAMING.md +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/examples/byoidc_bearer_token_example.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/examples/errors.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/examples/simple_append_only_streaming_query_example.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/examples/snapshot_mode_tuple_cursor_simple_example.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/examples/statement_properties_example.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/service.yml +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/auth.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/changelog_compressor.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/execution_mode.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/polling.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/result_readers.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/retry.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/src/confluent_sql/statement_properties.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/__init__.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/conftest.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_auth_unit.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_changelog_compressor_unit.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_changelog_unit.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_connection_unit_properties.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_connectors_unit.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_execution_mode_unit.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_polling_unit.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_result_readers_unit.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_retry_unit.py +0 -0
- {confluent_sql-0.5.5 → confluent_sql-0.7.0}/tests/unit/test_statement_properties_unit.py +0 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
---
|
|
2
|
+
paths: ["**"]
|
|
3
|
+
---
|
|
4
|
+
# Repo Software Engineering Rules
|
|
5
|
+
|
|
6
|
+
## Claude / Human Interactions
|
|
7
|
+
1. PR descriptions can be drafted by Claude, but should always be revised and finalized by a human before submission. PRs over-laden with automated content without human review are highly discouraged.
|
|
8
|
+
2. Claude should never directly reply to humans in contexts such as PR comments, etc. A PR is an artifact by humans for humans with agents assisting in the process. It is demeaning for humans at PR to be interacting with someone else's Claude, be they the PR author or a reviewer. Claude should never directly comment on a PR using "/code-review --comment" or other tooling.
|
|
9
|
+
3. Agents may be asked to author new issue descriptions, but these should always be reviewed and finalized by a human before submission, and the associated PR description should definitely err on human-authored content, never fully relying on agent-generated content.
|
|
10
|
+
4. Humans should always validate claude findings when reviewing PRs, issue descriptions, or any other content generated by agents, ensuring accuracy and relevance before taking any action based on those findings, then decide the subset to describe to the human audience, using their own language to describe the issue. Claudeisms should not make it into PR conversations.
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
## Pull Request Lifecycle Rules
|
|
14
|
+
### Branches must adhere to the following lifecycle rules:
|
|
15
|
+
1. A branch should be focused on a single logical change or feature. More complex changes should be split into multiple branches and pull requests.
|
|
16
|
+
- At most a handful of small, pithy 'also seen and fixed or improved here' changes can be included, and must always be explicitly pointed out in the pull request description.
|
|
17
|
+
2. A branch should have a human-readable reasonable number of lines of changes relative to its parent branch:
|
|
18
|
+
- At most 500-700 lines of actual primary codebase changes (excluding docs, docstrings, tests).
|
|
19
|
+
- Test suite and / or documentation changes can at most double the number of lines of primary codebase changes for a total maximum of around 2,100 lines including the non-primary codebase changes.
|
|
20
|
+
- package lock files (e.g., package-lock.json, uv.lock, other purely mechanical changes) are not counted towards the line limits.
|
|
21
|
+
- Exceptions to these limits must be explicitly justified in the pull request description, but should be truly exceptional.
|
|
22
|
+
3. Introducing new tooling, development practices, or changes to the development or CI/CD workflow should be done through consensus and discussion, such as through Slack (citing the thread in the PR description) or an issue written by a separate team member.
|
|
23
|
+
|
|
24
|
+
### Pull requests must adhere to the following rules:
|
|
25
|
+
1. A pull request should be raised only after the branch is up-to-date with its parent branch and passing unit tests and linting.
|
|
26
|
+
2. Do not raise, or advise raising, a pull request for a branch that violates the branch lifecycle rules unless a strong justification is given.
|
|
27
|
+
- Instead, determine ways to bring the branch into compliance with the branch lifecycle rules, such as through splitting separate concerns into independent or stacked branches, compacting wordy separate tests into parameterized ones, better use of fixtures, etc.
|
|
28
|
+
- If it is not possible to bring the branch into compliance, ask the human to provide a strong justification in the pull request description.
|
|
29
|
+
3. When merging a multi-commit pull request, the human should hand-write the ultimate merge commit message, summarizing the changes in a clear and concise manner.
|
|
30
|
+
|
|
31
|
+
### PR Descriptions
|
|
32
|
+
1. A PR description should clearly outline the single logical change or feature being introduced.
|
|
33
|
+
2. Any small, pithy 'also seen and fixed or improved here' changes must be explicitly pointed out in the PR description.
|
|
34
|
+
3. The PR description should include any justifications for exceptions to the branch or PR lifecycle rules.
|
|
35
|
+
4. Breaking changes to existing released interfaces must be highlighted in the PR description (as well as the corresponding changelog entry).
|
|
36
|
+
5. The PR description should reference any related issues ("Closes #131"), previous PR numbers, or relevant slack threads.
|
|
37
|
+
6. PRs for branches other than main for new tip development should clearly indicate the target branch in the PR title: "[v0.3.x] Fix the thing", "[oauth-integration] Make epic progress"
|
|
38
|
+
7. Security related issues should be called out very clearly in the PR description and should be PR'd in isolated branches.
|
|
39
|
+
- This includes those for "maintenance" dependency updates.
|
|
40
|
+
- Maintenance dependency updates should never be combined with feature development changes. New features which require new dependencies should only introduce those new dependencies and not also update unrelated dependencies.
|
|
41
|
+
|
|
42
|
+
## Branching
|
|
43
|
+
1. The main branch is used for the tip of development. It must always pass unit and integration/functional tests.
|
|
44
|
+
2. Larger development epics which might leave main in temporary broken state should be developed against a child integration branch and then have rule-following PRs make that branch grow in reasonable chunks. The integration branch is then PR cycled back into main, bypassing the size rules, since every line of change in the integration branch has already been reviewed and approved through the smaller, rule-following PRs.
|
|
45
|
+
- The only way non-present-in-main code should be introduced into an integration branch is through a rule-following PR being approved.
|
|
46
|
+
- Other techniques may include pseudo-feature flags as implemented by environment variable or other opt-in mechanisms, as pre-determined by developer and target reviewers.
|
|
47
|
+
3. Release branches (`v0.5.x`) will be cut from main when preparing for a release.
|
|
48
|
+
- The initial PR cutting the release branch should have zero changes from main.
|
|
49
|
+
- "Final" testing, bugbashing, etc. should be performed on the release branch, with fixes following the normal branch/PR size rules.
|
|
50
|
+
- After deciding that a release is a "go", then the PR to into the release branch stamping the new version and changelog date header should be made, then the release actually made, then merge-back into main.
|
|
51
|
+
- Any hotfixes required after the release should be developed against the release branch *first*, verified, then patch release made, then finally the changes merged back into main, including the patch release changelog updates.
|
|
52
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
* @confluentinc/dbt-product @confluentinc/sql-clients
|
|
@@ -8,7 +8,6 @@ global_job_config:
|
|
|
8
8
|
prologue:
|
|
9
9
|
commands:
|
|
10
10
|
- checkout
|
|
11
|
-
- . vault-setup
|
|
12
11
|
- curl -LsSf https://astral.sh/uv/0.10.11/install.sh | sh
|
|
13
12
|
- uv sync --frozen
|
|
14
13
|
|
|
@@ -18,8 +17,8 @@ blocks:
|
|
|
18
17
|
jobs:
|
|
19
18
|
- name: Publish `confluent_sql` to PyPi
|
|
20
19
|
commands:
|
|
21
|
-
- export VERSION=$(grep '^
|
|
20
|
+
- export VERSION=$(grep '^__version__' src/confluent_sql/__version__.py | cut -d'"' -f2)
|
|
22
21
|
- echo "Publishing confluent-sql $VERSION to PyPi"
|
|
23
22
|
- uv build
|
|
24
|
-
-
|
|
23
|
+
- . vault-get-secret --secret-key=pypi_token --export-as=CONFLUENT_SQL_KEY pypi/confluent-sql
|
|
25
24
|
- uv publish -u __token__ -p "$CONFLUENT_SQL_KEY" dist/*
|
|
@@ -22,7 +22,7 @@ global_job_config:
|
|
|
22
22
|
- . vault-setup
|
|
23
23
|
- curl -LsSf https://astral.sh/uv/0.10.11/install.sh | sh
|
|
24
24
|
- uv sync --frozen
|
|
25
|
-
- export VERSION=$(grep '^
|
|
25
|
+
- export VERSION=$(grep '^__version__' src/confluent_sql/__version__.py | cut -d'"' -f2)
|
|
26
26
|
|
|
27
27
|
blocks:
|
|
28
28
|
- name: "Test"
|
|
@@ -35,13 +35,13 @@ When you submit a statement (query, DDL, or data ingestion job), it becomes a fi
|
|
|
35
35
|
|
|
36
36
|
Every statement progresses through these phases:
|
|
37
37
|
|
|
38
|
-
| Phase | Meaning | Can Fetch Results Through This Driver?
|
|
39
|
-
| ------------- | --------------------------------- |
|
|
40
|
-
| **PENDING** | Queued on server, not yet started | No
|
|
41
|
-
| **RUNNING** | Actively executing |
|
|
42
|
-
| **COMPLETED** | Finished successfully | Yes (for both query/cursor types)
|
|
43
|
-
| **FAILED** | Error during execution | No
|
|
44
|
-
| **STOPPED** | Explicitly stopped through API. | No
|
|
38
|
+
| Phase | Meaning | Can Fetch Results Through This Driver? |
|
|
39
|
+
| ------------- | --------------------------------- | ------------------------------------------------ |
|
|
40
|
+
| **PENDING** | Queued on server, not yet started | No |
|
|
41
|
+
| **RUNNING** | Actively executing | Depends on statement kind and traits (see below) |
|
|
42
|
+
| **COMPLETED** | Finished successfully | Yes (for both query/cursor types) |
|
|
43
|
+
| **FAILED** | Error during execution | No |
|
|
44
|
+
| **STOPPED** | Explicitly stopped through API. | No |
|
|
45
45
|
|
|
46
46
|
### How Phases Progress
|
|
47
47
|
|
|
@@ -51,16 +51,22 @@ Every statement progresses through these phases:
|
|
|
51
51
|
PENDING → RUNNING → COMPLETED
|
|
52
52
|
```
|
|
53
53
|
|
|
54
|
-
When you call `cursor.execute()` for a snapshot query / default cursor, the driver
|
|
54
|
+
When you call `cursor.execute()` for a snapshot query / default cursor, the driver submits the
|
|
55
|
+
statement (HTTP POST) and polls until results can be fetched. Whether that's as soon as RUNNING
|
|
56
|
+
or only once COMPLETED depends on the statement's kind and traits, using the exact same logic as
|
|
57
|
+
streaming mode:
|
|
55
58
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
+
- A plain query with a finite, append-only result (e.g. a projection or filter with no `GROUP BY`)
|
|
60
|
+
is ready as soon as the statement reaches RUNNING — the server computes snapshot results as one
|
|
61
|
+
batch with nothing left to retract, so the result set is already stable by then.
|
|
62
|
+
- A bounded, non-append-only query (an aggregation that could still retract rows as it settles)
|
|
63
|
+
waits for COMPLETED, since its result isn't guaranteed final until then.
|
|
64
|
+
- A statement with no result set at all — `INSERT INTO`, or a `CREATE TABLE ... AS SELECT` — is a
|
|
65
|
+
genuinely finite write/population job in snapshot mode and also waits for COMPLETED, since its
|
|
66
|
+
side effect isn't guaranteed to have landed until then.
|
|
59
67
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
_NOTE_: This behavior may change in future releases, moving to have `.execute()` return once RUNNING phase is entered,
|
|
63
|
-
moving the additional blocking to the fetching results interactions.
|
|
68
|
+
Either way, `.execute()` blocks until the statement is ready by that definition, then returns to
|
|
69
|
+
let you begin to interact with the cursor / results.
|
|
64
70
|
|
|
65
71
|
**Streaming queries:**
|
|
66
72
|
|
|
@@ -233,11 +239,14 @@ print(f"Append-only: {stmt.is_append_only}") # True if statement results change
|
|
|
233
239
|
|
|
234
240
|
### Snapshot Mode
|
|
235
241
|
|
|
236
|
-
Snapshot queries are **blocking**—your code pauses until
|
|
242
|
+
Snapshot queries are **blocking**—your code pauses until results are ready to fetch. For most
|
|
243
|
+
queries (a finite, append-only result), that's as soon as the statement reaches `RUNNING`, since
|
|
244
|
+
the result set is already stable by then; a bounded, non-append-only aggregation instead waits for
|
|
245
|
+
`COMPLETED`, since its result could still change:
|
|
237
246
|
|
|
238
247
|
```python
|
|
239
248
|
cursor = connection.cursor()
|
|
240
|
-
cursor.execute("SELECT * FROM large_table") # ← Blocks here until
|
|
249
|
+
cursor.execute("SELECT * FROM large_table") # ← Blocks here until results can be fetched.
|
|
241
250
|
rows = cursor.fetchall()
|
|
242
251
|
```
|
|
243
252
|
|
|
@@ -2,12 +2,54 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this dbapi driver will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## 0.7.0, 2026-10-19
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- `Connection.dry_run_statement(statement_text, *, mode=ExecutionMode.SNAPSHOT, compute_pool_id=None)`: validate a statement with Flink's `sql.dry-run` in a single request, without running or storing it, and return the resulting `Statement`. The returned `Statement`'s `schema` and `is_append_only` may then be of interest to the caller. Raises `OperationalError` with the server's detail if Flink rejects the statement. (#246)
|
|
10
|
+
- New property `Statement.is_dry_run`: whether the statement was submitted with the `sql.dry-run` statement property. (#246)
|
|
11
|
+
- Support for the Flink Artifact API (`/artifact/v1/flink-artifacts` and `/presigned-upload-url`), for uploading UDF packages (a JAR, or for Python a ZIP) to Confluent Cloud.
|
|
12
|
+
See `examples/artifact_upload_example.py` for a full example of packaging, uploading and calling a Python UDF artifact end to end.
|
|
13
|
+
Artifact calls authenticate with a global key, or a dedicated Cloud resource management key via the new `cloud_api_key`/`cloud_api_secret` parameters of `connect()`.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- `Cursor.execute()` rejects `sql.dry-run` property being supplied as True with an `InterfaceError` pointing to use `Connection.dry_run_statement()` instead. Previously, valid SQL raised `StatementNotFoundError` (the cursor polled for a statement the server never stores) and invalid SQL raised `OperationalError`. (#246)
|
|
18
|
+
|
|
19
|
+
## 0.6.0, 2026-09-29
|
|
20
|
+
|
|
21
|
+
### Changed - Breaking
|
|
22
|
+
|
|
23
|
+
- `Connection.enable_tableflow()`: Renamed the `tableflow_formats` parameter to `table_formats`. Update calls from `enable_tableflow(tableflow_formats=...)` to `enable_tableflow(table_formats=...)`. (#214)
|
|
24
|
+
- `TableflowTopicSpec.config`: Now parsed as a typed `TableflowTopicConfig` instead of being left as a raw mapping. If you need unmodeled wire fields, read `topic.spec.raw.get("config")` instead. (#214)
|
|
25
|
+
|
|
26
|
+
### Added
|
|
27
|
+
|
|
28
|
+
- Support for enabling TableFlow against GCP/GCS. Use kind=`"GoogleCloudStorage"` and new related class `GcsStorage`, peer to ByobAwsStorage/AzureAdlsStorage/ManagedStorage. See new example `examples/tableflow_gcs_example.py` for more details. (#239)
|
|
29
|
+
- Ability to adjust an existing TableFlow configuration: `Connection.update_tableflow(table_name, *, table_formats=None, config=None, wait_for_running=True, timeout=300)` updates an already-enabled Tableflow topic's `table_formats`/`config` in place via `PATCH /tableflow/v1/tableflow-topics/{display_name}`, avoiding the disable/re-enable cycle `enable_tableflow` would otherwise require. `None` (the default) means "leave unchanged," for both of this method's own arguments and for each of `config`'s own sub-fields. `storage`/`display_name` remain immutable and have no in-place path; a caller needing to change either must recreate the topic. By default blocks until the topic returns to `RUNNING`; pass `wait_for_running=False` to return as soon as the update is accepted. Raises `TableflowTopicNotFoundError` if Tableflow isn't enabled for the table. (#214)
|
|
30
|
+
- Support for the new VARIANT type: VARIANT result columns are now decoded into typed Python values instead of raising `NotImplementedError`. An OBJECT becomes a `dict`, an ARRAY a `list`, and each scalar its matching Python type (`bool`, `int`, `float`, `Decimal`, `str`, `date`, `time`, `datetime`, `bytes`). Nanosecond-precision timestamps are truncated to microseconds. An undecodable node is returned as the new `UndecodableVariant`. VARIANT is not yet supported as a statement parameter (Flink has no VARIANT literal syntax).
|
|
31
|
+
- `Statement.warnings` exposes non-fatal warnings (`severity`, `created_at`, `reason`, `message`) reported by the server for a statement, parsed from `status.warnings` into a list of `StatementWarning`. `Cursor.warnings` is a convenience that delegates to `cursor.statement.warnings`, returning an empty list before a statement has been executed; unlike most cursor state, it remains readable after `Cursor.close()`. (#234)
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
|
|
35
|
+
- Type conversion methods across `types.py` now consistently raise dbapi-mandated exceptions (`DataError`, `InterfaceError`) rather than a bare builtin (`ValueError`, `decimal.InvalidOperation`) for problems with a Flink response value, a Python value that can't be represented as a Flink SQL literal, or a converter misconfigured with the wrong column type. (#204)
|
|
36
|
+
- `stop_statement(wait_for_stopped=True)` (the default) on a statement that had already reached FAILED on its own -- before the stop was ever requested -- no longer raises `OperationalError`. The blocking wait mistook "the statement is currently FAILED" for "the statement transitioned to FAILED while we were waiting for it to stop," when the former is really the same "already terminal, nothing to stop" success the `Statement`-object short-circuit already returns for a cached terminal statement; only a transition *into* FAILED partway through the wait is a genuine failure to stop cleanly. (#203)
|
|
37
|
+
|
|
38
|
+
### Changed
|
|
39
|
+
|
|
40
|
+
- Pushed the remaining `connect()`-only validation (`environment_id` and the endpoint-vs-`cloud_provider`/`cloud_region` requirement) down into `Connection.__init__()` as the single source of truth, so `connect()` and direct `Connection()` construction validate uniformly. `connect()` callers see no new errors; direct `Connection()` construction now fails fast with the same `InterfaceError`s (and the more specific per-field message for missing cloud info) instead of deferring to a confusing server-side failure later. (#213)
|
|
41
|
+
- **Behavior change for direct `Connection()` construction:** `organization_id` validation is now centralized in `Connection.__init__()` as well, extending the same fail-fast treatment to the constructor. Constructing a `Connection` directly with an empty `organization_id` and no global key now raises `InterfaceError("Organization ID is required")` rather than being silently accepted, and the `organization_id` property's former "no global key → return `""`" fallback has been removed as now-unreachable.
|
|
42
|
+
- `Statement.can_fetch_results()`'s snapshot-mode branch is now kind/trait-based (schema presence, `is_pure_ddl`, `is_bounded`, `is_append_only`) instead of unconditionally waiting for a terminal phase -- the same logic streaming mode already used. A bounded, append-only snapshot query that actually produces a result set (e.g. a plain projection) is now reported ready as soon as the statement reaches `RUNNING`, instead of blocking `Cursor.execute()` / `Connection.execute_snapshot_ddl()` until `COMPLETED`. Bounded, non-append-only snapshot queries (aggregations that could still retract) and any statement with no result schema at all -- `INSERT INTO`, snapshot CTAS (`CREATE TABLE ... AS SELECT`), and other DML/DDL that produces no rows -- are unaffected and still wait for terminal, since a finite write or population job isn't guaranteed to have landed until then (unlike its streaming counterpart, which may run forever and is ready once RUNNING). This can make `Cursor.execute()` and iteration/`fetchall()` return sooner for snapshot-mode queries that produce a result set. (#205)
|
|
43
|
+
|
|
44
|
+
### Removed
|
|
45
|
+
|
|
46
|
+
- The `dbname` parameter of `connect()`, deprecated in favor of `database` since 0.2.0, has been removed. Passing `dbname=` now raises `TypeError` for an unexpected keyword argument instead of emitting a `DeprecationWarning`. Use `database=` instead.
|
|
5
47
|
|
|
6
48
|
## 0.5.5, 2026-09-09
|
|
7
49
|
|
|
8
50
|
### Fixed
|
|
9
51
|
|
|
10
|
-
-
|
|
52
|
+
- `TableflowTopicConfig.to_spec()` now serializes `retention_ms`/`data_retention_ms` as strings rather than ints -- the API rejects non-string values for both, since the schema types them as int64, string-encoded to dodge JS/IEEE-754 double precision loss. (#215)
|
|
11
53
|
|
|
12
54
|
## 0.5.4, 2026-09-08
|
|
13
55
|
|
|
@@ -33,8 +33,9 @@ For comprehensive details on streaming queries, polling patterns, and changelog
|
|
|
33
33
|
|
|
34
34
|
- [Result Format Extensions](#result-format-extensions) - Dictionary rows, custom types
|
|
35
35
|
- [Streaming Query Support](#streaming-query-support) - Comprehensive streaming guide
|
|
36
|
-
- [Statement Lifecycle Management](#statement-lifecycle-management) - DDL, naming, stopping, deletion
|
|
36
|
+
- [Statement Lifecycle Management](#statement-lifecycle-management) - DDL, dry-run, naming, stopping, deletion
|
|
37
37
|
- [Tableflow Lifecycle](#tableflow-lifecycle) - Enable, read, and disable Iceberg/Delta sinks
|
|
38
|
+
- [Flink Artifacts](#flink-artifacts) - Upload and manage UDF packages
|
|
38
39
|
- [Introspection and Metadata](#introspection-and-metadata) - Properties for query state
|
|
39
40
|
- [Performance Monitoring](#performance-monitoring) - Fetch metrics
|
|
40
41
|
- [Type System Extensions](#type-system-extensions) - Flink type support
|
|
@@ -404,6 +405,36 @@ For more details on managing named and labeled statements, see the [Statement Na
|
|
|
404
405
|
|
|
405
406
|
---
|
|
406
407
|
|
|
408
|
+
### Dry-Run Validation (`dry_run_statement()`)
|
|
409
|
+
|
|
410
|
+
Validate a statement without running it, using Flink's `sql.dry-run`. Flink validates and plans
|
|
411
|
+
the statement and answers in the submission response, without storing it, so this is a single
|
|
412
|
+
request with nothing to clean up:
|
|
413
|
+
|
|
414
|
+
```python
|
|
415
|
+
statement = connection.dry_run_statement(
|
|
416
|
+
"SELECT id, price FROM orders", mode=ExecutionMode.STREAMING_QUERY
|
|
417
|
+
)
|
|
418
|
+
print(statement.schema) # the query's result columns; None for DDL
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
- `mode` (default `ExecutionMode.SNAPSHOT`) sets `sql.snapshot.mode` as a cursor in that mode
|
|
422
|
+
would. The result schema is the same in either mode, but the statement's traits are not:
|
|
423
|
+
`is_bounded` and `is_append_only` describe a run in the mode you validated in. To learn whether
|
|
424
|
+
a streaming submission would be append-only (e.g. an unbounded aggregation is not), pass
|
|
425
|
+
`ExecutionMode.STREAMING_QUERY`; under the default, a snapshot's bounded result always reports
|
|
426
|
+
append-only.
|
|
427
|
+
- Invalid SQL raises `OperationalError` with the server's detail.
|
|
428
|
+
- The returned `Statement` has `is_dry_run` true and `statement_id` `""`. It was never stored,
|
|
429
|
+
so `get_statement()` for it raises `StatementNotFoundError`.
|
|
430
|
+
- The statement text is sent as-is (no parameter interpolation).
|
|
431
|
+
|
|
432
|
+
`Cursor.execute()` does not accept property `sql.dry-run`: a dry-run has no rows to fetch and no
|
|
433
|
+
server-side statement to manage, so it doesn't fit the cursor lifecycle. Passing it as true raises
|
|
434
|
+
`InterfaceError`, pointing to use this `Connection`-level method instead.
|
|
435
|
+
|
|
436
|
+
---
|
|
437
|
+
|
|
407
438
|
### Statement Naming and Labeling
|
|
408
439
|
|
|
409
440
|
Control statement identity and grouping for tracking and management.
|
|
@@ -569,7 +600,7 @@ cursor.delete_statement()
|
|
|
569
600
|
## Tableflow Lifecycle
|
|
570
601
|
|
|
571
602
|
[Tableflow](https://www.confluent.io/product/tableflow/) materializes the Kafka topic backing a
|
|
572
|
-
Flink table into an Iceberg or Delta table.
|
|
603
|
+
Flink table into an Iceberg or Delta table. Four `Connection` methods manage that sink. Enabling
|
|
573
604
|
it also unlocks
|
|
574
605
|
[efficiency gains for snapshot queries](https://docs.confluent.io/cloud/current/flink/concepts/snapshot-queries.html#snapshot-queries-and-tableflow)
|
|
575
606
|
against the table.
|
|
@@ -590,13 +621,13 @@ A runnable example covering the full enable/get/disable lifecycle is in
|
|
|
590
621
|
|
|
591
622
|
There is a single format vocabulary, `TableFormat` (`ICEBERG` / `DELTA`), on both the request and
|
|
592
623
|
response sides. A topic can carry **both** formats at once (there is no per-format config), so
|
|
593
|
-
`enable_tableflow`'s `
|
|
624
|
+
`enable_tableflow`'s `table_formats` argument accepts either a single `TableFormat` for the
|
|
594
625
|
common case or a collection for both:
|
|
595
626
|
|
|
596
627
|
```python
|
|
597
|
-
connection.enable_tableflow("orders",
|
|
628
|
+
connection.enable_tableflow("orders", table_formats=TableFormat.ICEBERG, storage=...)
|
|
598
629
|
connection.enable_tableflow(
|
|
599
|
-
"orders",
|
|
630
|
+
"orders", table_formats={TableFormat.ICEBERG, TableFormat.DELTA}, storage=...
|
|
600
631
|
)
|
|
601
632
|
```
|
|
602
633
|
|
|
@@ -606,7 +637,7 @@ plain set comparison:
|
|
|
606
637
|
|
|
607
638
|
```python
|
|
608
639
|
topic = connection.enable_tableflow(
|
|
609
|
-
"orders",
|
|
640
|
+
"orders", table_formats={TableFormat.ICEBERG, TableFormat.DELTA},
|
|
610
641
|
storage=ManagedStorage(), wait_for_running=True,
|
|
611
642
|
)
|
|
612
643
|
assert set(topic.spec.table_formats) == {TableFormat.ICEBERG, TableFormat.DELTA}
|
|
@@ -620,6 +651,8 @@ assert set(topic.spec.table_formats) == {TableFormat.ICEBERG, TableFormat.DELTA}
|
|
|
620
651
|
- `ByobAwsStorage(bucket_name=..., provider_integration_id=...)` — bring-your-own AWS S3 bucket.
|
|
621
652
|
- `AzureAdlsStorage(storage_account_name=..., container_name=..., provider_integration_id=...)` —
|
|
622
653
|
customer-owned Azure Data Lake Storage Gen2.
|
|
654
|
+
- `GcsStorage(bucket_name=..., provider_integration_id=...)` — customer-owned Google Cloud Storage
|
|
655
|
+
bucket.
|
|
623
656
|
|
|
624
657
|
### Cluster-id resolution
|
|
625
658
|
|
|
@@ -638,7 +671,7 @@ from confluent_sql import ManagedStorage, TableFormat, TableflowPhase
|
|
|
638
671
|
|
|
639
672
|
topic = connection.enable_tableflow(
|
|
640
673
|
"orders",
|
|
641
|
-
|
|
674
|
+
table_formats=TableFormat.ICEBERG,
|
|
642
675
|
storage=ManagedStorage(),
|
|
643
676
|
)
|
|
644
677
|
assert topic.phase is TableflowPhase.RUNNING # blocked to RUNNING by default
|
|
@@ -646,7 +679,7 @@ assert topic.phase is TableflowPhase.RUNNING # blocked to RUNNING by default
|
|
|
646
679
|
|
|
647
680
|
**Behavior notes:**
|
|
648
681
|
|
|
649
|
-
- `
|
|
682
|
+
- `table_formats` and `storage` are required (no defaults); `table_formats` must name at
|
|
650
683
|
least one format. `config` is an optional `TableflowTopicConfig` (retention, error-handling)
|
|
651
684
|
shared across all enabled formats.
|
|
652
685
|
- Blocks until `RUNNING` by default (`wait_for_running=True`), raising `OperationalError` on
|
|
@@ -667,6 +700,38 @@ print(topic.spec.table_formats) # [TableFormat.ICEBERG, ...]
|
|
|
667
700
|
Raises `TableflowTopicNotFoundError` if Tableflow is not enabled for the topic (HTTP 404). There is
|
|
668
701
|
no separate health check — health is read off `get_tableflow(...).phase`.
|
|
669
702
|
|
|
703
|
+
### `update_tableflow()` — change formats/config on an existing sink
|
|
704
|
+
|
|
705
|
+
```python
|
|
706
|
+
from confluent_sql import TableflowTopicConfig, TableFormat, TableflowPhase
|
|
707
|
+
|
|
708
|
+
topic = connection.update_tableflow(
|
|
709
|
+
"orders",
|
|
710
|
+
table_formats={TableFormat.ICEBERG, TableFormat.DELTA},
|
|
711
|
+
config=TableflowTopicConfig(retention_ms="1209600000"),
|
|
712
|
+
)
|
|
713
|
+
assert topic.phase is TableflowPhase.RUNNING # blocked to RUNNING by default
|
|
714
|
+
```
|
|
715
|
+
|
|
716
|
+
Lets a caller change only `table_formats`/`config` on an already-enabled topic — a `PATCH`, not a
|
|
717
|
+
disable/re-enable — so an in-place config change doesn't drop and rebuild the sink. `storage` and
|
|
718
|
+
the table name itself are immutable server-side and have no in-place path; changing either requires
|
|
719
|
+
`disable_tableflow` + `enable_tableflow` instead.
|
|
720
|
+
|
|
721
|
+
**Behavior notes:**
|
|
722
|
+
|
|
723
|
+
- `table_formats` and `config` are each independently optional, but at least one non-empty update must be given.
|
|
724
|
+
`None` means "leave unchanged" — both for the method's own two arguments and for each field on
|
|
725
|
+
`config` (`retention_ms`, `data_retention_ms`, `error_handling`). A field can't be *cleared* this
|
|
726
|
+
way: every `config` field has a server-enforced default and the server rejects an explicit null.
|
|
727
|
+
- When `table_formats` is given, it replaces the full list rather than adding to it.
|
|
728
|
+
- This method sends whatever it's given — it does not diff against the topic's current state.
|
|
729
|
+
Compare against a prior `get_tableflow()` yourself if you only want to send what actually changed.
|
|
730
|
+
- Blocks until `RUNNING` by default (`wait_for_running=True`), raising `OperationalError` on
|
|
731
|
+
`FAILED`, same as `enable_tableflow`. Pass `wait_for_running=False` to return as soon as the
|
|
732
|
+
update is accepted.
|
|
733
|
+
- Raises `TableflowTopicNotFoundError` if Tableflow is not enabled for the topic (HTTP 404).
|
|
734
|
+
|
|
670
735
|
### `disable_tableflow()` — tear down the sink
|
|
671
736
|
|
|
672
737
|
```python
|
|
@@ -699,7 +764,7 @@ config = TableflowTopicConfig(retention_ms="604800000")
|
|
|
699
764
|
for table in ("orders", "shipments", "returns"):
|
|
700
765
|
connection.enable_tableflow(
|
|
701
766
|
table,
|
|
702
|
-
|
|
767
|
+
table_formats=TableFormat.ICEBERG,
|
|
703
768
|
storage=storage,
|
|
704
769
|
config=config,
|
|
705
770
|
)
|
|
@@ -707,6 +772,59 @@ for table in ("orders", "shipments", "returns"):
|
|
|
707
772
|
|
|
708
773
|
---
|
|
709
774
|
|
|
775
|
+
## Flink Artifacts
|
|
776
|
+
|
|
777
|
+
Flink artifacts are how user code -- a UDF packaged as a JAR (Java) or a ZIP (Python) -- gets into
|
|
778
|
+
Confluent Cloud Flink. Upload one, then register a function from it with
|
|
779
|
+
`CREATE FUNCTION ... USING JAR 'confluent-artifact://<artifact id>'`. Artifacts are scoped to the
|
|
780
|
+
connection's cloud, region and environment (from `connect()`'s `cloud_provider`/`cloud_region`, or a
|
|
781
|
+
standard `https://flink.<region>.<cloud>.confluent.cloud` `endpoint`).
|
|
782
|
+
|
|
783
|
+
A runnable end-to-end example -- build a Python UDF, upload it, call it from SQL, clean up -- is in
|
|
784
|
+
[examples/artifact_upload_example.py](examples/artifact_upload_example.py).
|
|
785
|
+
|
|
786
|
+
> **Credentials.** Artifacts need a global API key, or a "Cloud resource management" key passed as
|
|
787
|
+
> `cloud_api_key`/`cloud_api_secret` (used only when no global key is given). A Tableflow or Flink
|
|
788
|
+
> key won't work, and neither will BYOIDC -- like Tableflow, artifacts are a control-plane surface.
|
|
789
|
+
|
|
790
|
+
The artifact operations live on an `ArtifactsApi` object, returned by
|
|
791
|
+
`Connection.get_artifacts_api()` (composed on first call, then cached):
|
|
792
|
+
|
|
793
|
+
```python
|
|
794
|
+
from confluent_sql import ArtifactRuntimeLanguage
|
|
795
|
+
|
|
796
|
+
artifacts = connection.get_artifacts_api()
|
|
797
|
+
|
|
798
|
+
artifact = artifacts.create(
|
|
799
|
+
"my-udf",
|
|
800
|
+
"dist/my_udf-0.1.0.zip", # path of the archive to upload
|
|
801
|
+
runtime_language=ArtifactRuntimeLanguage.PYTHON,
|
|
802
|
+
description="T-shirt size comparisons",
|
|
803
|
+
)
|
|
804
|
+
artifacts.get(artifact.id).versions # versions appear on a read, not on create/list
|
|
805
|
+
artifacts.list(runtime_language="PYTHON")
|
|
806
|
+
artifacts.delete(artifact.id) # blocks until gone; wait_for_removal=False to opt out
|
|
807
|
+
```
|
|
808
|
+
|
|
809
|
+
`create` requests a presigned URL, uploads the file to it and registers the artifact. It infers
|
|
810
|
+
`content_format` from the `.jar`/`.zip` extension when you don't pass one. Only JAVA with JAR and PYTHON
|
|
811
|
+
with ZIP are accepted; anything else raises `InterfaceError` before the upload. `runtime_language`
|
|
812
|
+
defaults to JAVA, so a ZIP needs `runtime_language=PYTHON`. The file is streamed
|
|
813
|
+
from disk.
|
|
814
|
+
|
|
815
|
+
Uploads retry failures to *connect* (nothing sent yet) three times by default, but not failures
|
|
816
|
+
mid-transfer, which would re-send the whole archive. Tune with an `UploadRetry` via
|
|
817
|
+
`create(upload_retry=...)`: `UploadRetry(connect_retries=3, transfer_retries=0)`.
|
|
818
|
+
|
|
819
|
+
The Cloud API doesn't currently support adding versions to an existing artifact: each artifact has
|
|
820
|
+
a single server-assigned version, and its `display_name` is immutable. To ship new code, create a
|
|
821
|
+
new artifact rather than updating.
|
|
822
|
+
|
|
823
|
+
Errors: `ArtifactNotFoundError` (404 on get/delete), `ArtifactAlreadyExistsError` (display
|
|
824
|
+
name taken). Python UDFs run only in streaming mode.
|
|
825
|
+
|
|
826
|
+
---
|
|
827
|
+
|
|
710
828
|
## Introspection and Metadata
|
|
711
829
|
|
|
712
830
|
### Connection Properties
|
|
@@ -786,6 +904,7 @@ The `cursor.statement` property provides detailed metadata about the executed qu
|
|
|
786
904
|
| `is_append_only` | `bool` | Query produces only inserts (vs changelog with updates/deletes) |
|
|
787
905
|
| `is_bounded` | `bool` | Query has finite result set (snapshot) vs unbounded (streaming) |
|
|
788
906
|
| `is_deletable` | `bool` | Statement can be deleted |
|
|
907
|
+
| `is_dry_run` | `bool` | Statement was submitted with `sql.dry-run` (validated, never stored) |
|
|
789
908
|
| `schema` | `Schema` | Result schema with column names and types |
|
|
790
909
|
| `sql_kind` | `str` | Query type: `SELECT`, `INSERT`, `CREATE`, etc. |
|
|
791
910
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: confluent-sql
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: DB-API v2 compliant driver for Confluent Cloud Flink SQL
|
|
5
5
|
Project-URL: Repository, https://github.com/confluentinc/confluent-sql
|
|
6
6
|
Project-URL: Documentation, https://github.com/confluentinc/confluent-sql?tab=readme-ov-file#confluent-sql
|
|
@@ -227,6 +227,7 @@ Requires-Dist: httpx<0.29.0,>=0.23.0
|
|
|
227
227
|
Provides-Extra: dev
|
|
228
228
|
Requires-Dist: pytest; extra == 'dev'
|
|
229
229
|
Requires-Dist: pytest-cov; extra == 'dev'
|
|
230
|
+
Requires-Dist: pytest-xdist>=3.5.0; extra == 'dev'
|
|
230
231
|
Description-Content-Type: text/markdown
|
|
231
232
|
|
|
232
233
|
# confluent-sql
|
|
@@ -24,6 +24,7 @@ At time of writing, Flink is not able to infer the type of a SQL literal based o
|
|
|
24
24
|
| `MAP` | `dict` | `MAP['k1', 1, 'k2', 2]` | Flink MAPs are strongly typed on both the key and value type, so heterogeneous Python dicts are disallowed (outside of `None` keys or values, if the MAP key or value type includes nullability). Provided `dict`s cannot contain only-`None` keys or values, due to the need to infer the type for the corresponding key or value `NULL`. |
|
|
25
25
|
| `MULTISET` | `collections.Counter` | _(n/a)_ | Driver can deserialize results into `Counter`, but Flink cannot accept MULTISET literals, so submitting `Counter` parameters is unsupported. Any supported Python type corresponding to the MULTISET's key type is supported. |
|
|
26
26
|
| `ROW` | `tuple`, `namedtuple`, `typing.NamedTuple`, `@dataclass` | `ROW(1, 'foo')` | Parameter `ROW` values require `tuple`, `namedtuple`, `NamedTuple` or `@dataclass` instances, and are transcribed to literal `ROW` values positionally. Statement results containing `ROW` members will by default be mapped to auto-generated `collections.namedtuple` classes with named fields. Users may register their own `collections.namedtuple`, `typing.NamedTuple`, or `@dataclass` for use for `ROW` result mapping using the `connection.register_row_type()` method. |
|
|
27
|
+
| `VARIANT` | `dict` (OBJECT) / `list` (ARRAY) / typed scalar (`bool`, `int`, `float`, `Decimal`, `str`, `date`, `time`, `datetime`, `bytes`) / `UndecodableVariant` | _(n/a)_ | VARIANT values are encountered in results in two broad ways: from an explicit `PARSE_JSON()` (or similar) call within the statement itself, or when an underlying Kafka topic's serialization format maps onto Flink's VARIANT type — for example a JSON Schema carrying an open "{}" field. Result decoding only. A VARIANT toplevel OBJECT or member becomes a `dict`, an ARRAY a `list`, and each scalar its matching Python type (`bool`, `int`, `float`, `Decimal`, `str`, `date`, `time`, `datetime`, `bytes`). Nanosecond timestamps are truncated to microseconds (Python `datetime` limit). An undecodable node is returned as `UndecodableVariant`. Cannot be sent as a statement parameter, as Flink has no VARIANT literal syntax yet. |
|
|
27
28
|
| `NULL` | `None` | _(n/a)_ | Only produced by results; cannot be sent because Flink needs typed NULLs—use `SqlNone` instead. |
|
|
28
29
|
| _(any typed NULL)_ | `SqlNone` | `cast(null as INTEGER)` | `SqlNone` enforces Flink type names; parameterized types must match regex (e.g., `ROW<...>`). Results never return `SqlNone`, so mapping is one-way. |
|
|
29
30
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Minimal project used by examples/artifact_upload_example.py: `uv build --sdist` of this
|
|
2
|
+
# directory produces the tarball the example zips and uploads as a Python UDF artifact.
|
|
3
|
+
[project]
|
|
4
|
+
name = "example-udf"
|
|
5
|
+
version = "0.1.0"
|
|
6
|
+
description = "Example Python UDF packaged for Confluent Cloud Flink"
|
|
7
|
+
requires-python = "~=3.11.0"
|
|
8
|
+
dependencies = ["apache-flink==2.0.0", "setuptools<81"]
|
|
9
|
+
|
|
10
|
+
[build-system]
|
|
11
|
+
requires = ["uv_build>=0.11.30,<0.12.0"]
|
|
12
|
+
build-backend = "uv_build"
|
|
File without changes
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# pyright: reportMissingImports=false
|
|
2
|
+
# (pyflink is only available in the Confluent Cloud UDF runtime, not this project's environment)
|
|
3
|
+
from pyflink.table import DataTypes
|
|
4
|
+
from pyflink.table.types import DataType
|
|
5
|
+
from pyflink.table.udf import udf
|
|
6
|
+
|
|
7
|
+
# Dictionary mapping both full names and abbreviations to numeric size values
|
|
8
|
+
_SIZE_MAP = {
|
|
9
|
+
"x-small": 0,
|
|
10
|
+
"xs": 0,
|
|
11
|
+
"small": 1,
|
|
12
|
+
"s": 1,
|
|
13
|
+
"medium": 2,
|
|
14
|
+
"m": 2,
|
|
15
|
+
"large": 3,
|
|
16
|
+
"l": 3,
|
|
17
|
+
"x-large": 4,
|
|
18
|
+
"xl": 4,
|
|
19
|
+
"xx-large": 5,
|
|
20
|
+
"xxl": 5,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _get_size_value(shirt: str) -> int:
|
|
25
|
+
"""
|
|
26
|
+
Returns the numeric size value for a given shirt size string.
|
|
27
|
+
Returns -1 if the size is not found.
|
|
28
|
+
"""
|
|
29
|
+
if shirt is None:
|
|
30
|
+
return -1
|
|
31
|
+
return _SIZE_MAP.get(shirt.strip().lower(), -1)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _f_is_smaller(shirt1: str, shirt2: str) -> bool:
|
|
35
|
+
"""
|
|
36
|
+
Returns True if shirt1 is a smaller size than shirt2 based on standard T-shirt sizes.
|
|
37
|
+
If a size cannot be found, returns False.
|
|
38
|
+
"""
|
|
39
|
+
size1 = _get_size_value(shirt1)
|
|
40
|
+
size2 = _get_size_value(shirt2)
|
|
41
|
+
if size1 == -1 or size2 == -1:
|
|
42
|
+
return False
|
|
43
|
+
return size1 < size2
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
_is_smaller_inp_types: list[DataType] = [
|
|
47
|
+
DataTypes.STRING(),
|
|
48
|
+
DataTypes.STRING(),
|
|
49
|
+
]
|
|
50
|
+
is_smaller = udf(
|
|
51
|
+
_f_is_smaller,
|
|
52
|
+
input_types=_is_smaller_inp_types,
|
|
53
|
+
result_type=DataTypes.BOOLEAN(),
|
|
54
|
+
)
|