e6data-python-connector 2.3.15__tar.gz → 3.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {e6data_python_connector-2.3.15/e6data_python_connector.egg-info → e6data_python_connector-3.0.1}/PKG-INFO +249 -7
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/README.md +240 -5
- e6data_python_connector-3.0.1/e6data_python_connector/aio.py +26 -0
- e6data_python_connector-3.0.1/e6data_python_connector/async_cluster_manager.py +272 -0
- e6data_python_connector-3.0.1/e6data_python_connector/async_connection.py +620 -0
- e6data_python_connector-3.0.1/e6data_python_connector/async_connection_pool.py +429 -0
- e6data_python_connector-3.0.1/e6data_python_connector/async_cursor.py +923 -0
- e6data_python_connector-3.0.1/e6data_python_connector/async_dialect.py +285 -0
- e6data_python_connector-3.0.1/e6data_python_connector/async_oauth.py +248 -0
- e6data_python_connector-3.0.1/e6data_python_connector/async_work.py +105 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/cluster_manager.py +167 -11
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/common.py +2 -1
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/connection_pool.py +11 -9
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/datainputstream.py +43 -9
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/date_time_utils.py +4 -8
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/dialect.py +7 -2
- e6data_python_connector-3.0.1/e6data_python_connector/e6data_grpc.py +2699 -0
- e6data_python_connector-3.0.1/e6data_python_connector/exceptions.py +43 -0
- e6data_python_connector-3.0.1/e6data_python_connector/oauth.py +185 -0
- e6data_python_connector-3.0.1/e6data_python_connector/oauth_cache.py +238 -0
- e6data_python_connector-3.0.1/e6data_python_connector/oauth_common.py +66 -0
- e6data_python_connector-3.0.1/e6data_python_connector/result_batch.py +49 -0
- e6data_python_connector-3.0.1/e6data_python_connector/result_decode.py +559 -0
- e6data_python_connector-3.0.1/e6data_python_connector/result_decode_worker.py +50 -0
- e6data_python_connector-3.0.1/e6data_python_connector/result_prefetch.py +421 -0
- e6data_python_connector-3.0.1/e6data_python_connector/server/e6x_engine_pb2.py +209 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/server/e6x_engine_pb2_grpc.py +43 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1/e6data_python_connector.egg-info}/PKG-INFO +249 -7
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector.egg-info/SOURCES.txt +41 -5
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector.egg-info/entry_points.txt +1 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector.egg-info/requires.txt +7 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector.egg-info/top_level.txt +0 -1
- e6data_python_connector-3.0.1/pyproject.toml +12 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/setup.py +7 -2
- e6data_python_connector-3.0.1/test/conftest.py +18 -0
- e6data_python_connector-3.0.1/test/integration/__init__.py +1 -0
- e6data_python_connector-3.0.1/test/integration/benchmark_preflight.py +30 -0
- e6data_python_connector-3.0.1/test/integration/benchmark_result_batch_v2.py +149 -0
- e6data_python_connector-3.0.1/test/integration/benchmark_sync_async.py +177 -0
- e6data_python_connector-3.0.1/test/integration/expiry_preflight.py +37 -0
- e6data_python_connector-3.0.1/test/integration/result_batch_benchmark_support.py +196 -0
- e6data_python_connector-3.0.1/test/integration/support.py +176 -0
- e6data_python_connector-3.0.1/test/integration/test_async_connection.py +24 -0
- e6data_python_connector-3.0.1/test/integration/test_async_pool.py +51 -0
- e6data_python_connector-3.0.1/test/integration/test_async_queries.py +62 -0
- e6data_python_connector-3.0.1/test/integration/test_async_resume.py +26 -0
- e6data_python_connector-3.0.1/test/integration/test_async_sqlalchemy.py +45 -0
- e6data_python_connector-3.0.1/test/integration/test_async_token_lifecycle.py +35 -0
- e6data_python_connector-3.0.1/test/integration/test_e2e_auth_boundary.py +34 -0
- e6data_python_connector-3.0.1/test/integration/test_e2e_oauth_continuation.py +57 -0
- e6data_python_connector-3.0.1/test/integration/test_e2e_readonly_matrix.py +173 -0
- e6data_python_connector-3.0.1/test/integration/test_result_batch_v2.py +120 -0
- e6data_python_connector-3.0.1/test/integration/test_shared_token_issuance.py +97 -0
- e6data_python_connector-3.0.1/test/integration/test_sync_oauth_lifecycle.py +20 -0
- e6data_python_connector-3.0.1/test/integration/token_observation.py +55 -0
- e6data_python_connector-3.0.1/test/test_cluster_manager_efficiency.py +105 -0
- e6data_python_connector-3.0.1/test/test_cluster_manager_oauth.py +193 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_cluster_manager_strategy.py +1 -1
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_connection_pool.py +36 -20
- e6data_python_connector-3.0.1/test/test_oauth_authentication.py +592 -0
- e6data_python_connector-3.0.1/test/test_oauth_auto_resume.py +414 -0
- e6data_python_connector-3.0.1/test/test_oauth_resume_integration.py +244 -0
- e6data_python_connector-2.3.15/e6data_python_connector/e6data_grpc.py +0 -1630
- e6data_python_connector-2.3.15/e6data_python_connector/exceptions.py +0 -15
- e6data_python_connector-2.3.15/e6data_python_connector/server/e6x_engine_pb2.py +0 -207
- e6data_python_connector-2.3.15/gen-py/__init__.py +0 -0
- e6data_python_connector-2.3.15/gen-py/e6x_vector/__init__.py +0 -1
- e6data_python_connector-2.3.15/gen-py/e6x_vector/constants.py +0 -15
- e6data_python_connector-2.3.15/gen-py/e6x_vector/ttypes.py +0 -1875
- e6data_python_connector-2.3.15/pyproject.toml +0 -5
- e6data_python_connector-2.3.15/test/test_cluster_manager_efficiency.py +0 -198
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/LICENSE +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/MANIFEST.in +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/__init__.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/cluster_server/__init__.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/cluster_server/cluster_pb2.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/cluster_server/cluster_pb2_grpc.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/constants.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/e6x_vector/constants.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/e6x_vector/ttypes.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/server/QueryEngineService.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/server/__init__.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/server/constants.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/server/e6x_engine_pb2.pyi +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/server/ttypes.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/strategy.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector/typeId.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/e6data_python_connector.egg-info/dependency_links.txt +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/setup.cfg +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/__init__.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/analyze_38_nines.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/analyze_all_cases.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/analyze_binary.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/analyze_correct_value.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/analyze_fields.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/check_decimal_errors.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/cleanup_test_files.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/debug_38_nines.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/debug_binary.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/final_test.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/move_tests.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/quick_test.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_38_nines.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_all_decimal128_cases.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_cluster_manager_none_strategy.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_code_optimizations.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_comprehensive.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_connection_pool_e2e.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_current_implementation.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_decimal128_binary_parsing.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_decimal128_parsing.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_dialect_columns.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_fix.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_improved_parsing.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_known_case.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_manual_analysis.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_mock_server.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_multiprocessing_fix.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_new_implementation.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_pool_concurrency_simple.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_pool_threading_multiprocessing.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_simple_connection_pool.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_specific_binary.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_strategy.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_strategy_logic.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_strategy_persistence_fix.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_strategy_sharing_fix.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/test_user_binary_value.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/tests.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/tests_grpc.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/validate_decimal128.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/validate_implementation.py +0 -0
- {e6data_python_connector-2.3.15 → e6data_python_connector-3.0.1}/test/verify_decimal_fix.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.1
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
6
|
Author: e6data, Inc.
|
|
@@ -27,6 +27,11 @@ Requires-Dist: pytz>=2024.1
|
|
|
27
27
|
Requires-Dist: thrift>=0.20.0
|
|
28
28
|
Requires-Dist: grpcio>=1.65.1
|
|
29
29
|
Requires-Dist: grpcio-tools>=1.65.1
|
|
30
|
+
Provides-Extra: async
|
|
31
|
+
Requires-Dist: httpx<1,>=0.28.1; extra == "async"
|
|
32
|
+
Provides-Extra: async-sqlalchemy
|
|
33
|
+
Requires-Dist: httpx<1,>=0.28.1; extra == "async-sqlalchemy"
|
|
34
|
+
Requires-Dist: SQLAlchemy[asyncio]<2.1,>=2.0; extra == "async-sqlalchemy"
|
|
30
35
|
Dynamic: author
|
|
31
36
|
Dynamic: author-email
|
|
32
37
|
Dynamic: classifier
|
|
@@ -34,16 +39,38 @@ Dynamic: description
|
|
|
34
39
|
Dynamic: description-content-type
|
|
35
40
|
Dynamic: home-page
|
|
36
41
|
Dynamic: license
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
Dynamic: provides-extra
|
|
37
44
|
Dynamic: requires-dist
|
|
38
45
|
Dynamic: summary
|
|
39
46
|
|
|
40
47
|
# e6data Python Connector
|
|
41
48
|
|
|
42
|
-
|
|
49
|
+
Package version and extras are defined in `setup.py`.
|
|
43
50
|
|
|
44
51
|
## Introduction
|
|
45
52
|
|
|
46
|
-
The e6data Connector for Python provides an interface for writing Python applications that can connect to e6data and perform operations. It includes
|
|
53
|
+
The e6data Connector for Python provides an interface for writing Python applications that can connect to e6data and perform operations. It includes routing support for blue-green deployments. Active query handles remain tied to the engine that accepted them; failover does not guarantee uninterrupted results.
|
|
54
|
+
|
|
55
|
+
## Optional native asyncio API
|
|
56
|
+
|
|
57
|
+
This branch adds native async connection, cursor, pooling and SQLAlchemy support for Python 3.11+. Release qualification remains incomplete. Install this branch's built package with `[async]` or `[async-sqlalchemy]`; these instructions do not assert that the feature is published.
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from e6data_python_connector.aio import AsyncConnection
|
|
61
|
+
|
|
62
|
+
async def query(config, sql):
|
|
63
|
+
async with AsyncConnection(**config) as connection:
|
|
64
|
+
async with connection.cursor() as cursor:
|
|
65
|
+
await cursor.execute(sql)
|
|
66
|
+
return await cursor.fetchall()
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
`config` is application-supplied connection configuration. OAuth requires verified TLS. Native `fetchone()` preserves the one-row outer list, while SQLAlchemy `e6data+asyncio` adapts it to a normal row. Ambiguous submission and incomplete result errors must not be handled by blindly retrying the query. Use context managers for bounded cleanup.
|
|
70
|
+
|
|
71
|
+
See [Async API](docs/ASYNC_API.md) for the complete API map, deadlines, pooling, SQLAlchemy and test configuration, [OAuth lifecycle](docs/OAUTH_LIFECYCLE.md) for renewal and failure behavior, and [the application example](examples/async_query.py).
|
|
72
|
+
|
|
73
|
+
The new CI keeps the full-package coverage denominator and a greater-than-80% gate. The initial baseline was about 19%, so focused async test success is not a release pass. Real-service tests require explicit configuration and are qualified separately.
|
|
47
74
|
|
|
48
75
|
### Dependencies
|
|
49
76
|
Make sure to install below dependencies and wheel before install e6data-python-connector.
|
|
@@ -120,8 +147,8 @@ The `Connection` class supports the following parameters:
|
|
|
120
147
|
|-----------|------|----------|---------|-------------|
|
|
121
148
|
| `host` | str | Yes | - | IP address or hostname of the e6data cluster |
|
|
122
149
|
| `port` | int | Yes | - | Port of the e6data engine (typically 80) |
|
|
123
|
-
| `username` | str |
|
|
124
|
-
| `password` | str |
|
|
150
|
+
| `username` | str | Conditional | None | Your e6data Email ID. Required unless authenticating with OAuth |
|
|
151
|
+
| `password` | str | Conditional | None | Access Token generated in the e6data console. Required unless authenticating with OAuth |
|
|
125
152
|
| `database` | str | No | None | Database to perform queries on |
|
|
126
153
|
| `catalog` | str | No | None | Catalog name |
|
|
127
154
|
| `cluster_name` | str | No | None | Name of the cluster for cluster-specific operations |
|
|
@@ -131,6 +158,76 @@ The `Connection` class supports the following parameters:
|
|
|
131
158
|
| `grpc_options` | dict | No | None | Additional gRPC configuration options |
|
|
132
159
|
| `debug` | bool | No | False | Enable debug logging for troubleshooting |
|
|
133
160
|
| `require_fastbinary` | bool | No | True | Require fastbinary module for Thrift deserialization. Set to False to use pure Python implementation if system dependencies cannot be installed |
|
|
161
|
+
| `client_id` | str | No | None | OAuth 2.0 client id. Use with `client_secret` and `token_url` |
|
|
162
|
+
| `client_secret` | str | No | None | OAuth 2.0 client secret |
|
|
163
|
+
| `token_url` | str | No | None | Token endpoint of the authorization server |
|
|
164
|
+
| `oauth_scope` | str | No | None | Space-delimited scopes to request. Omit for the client's full registered set |
|
|
165
|
+
| `access_token` | str | No | None | A previously obtained access token, for callers that mint their own |
|
|
166
|
+
| `client_auth_method` | str | No | `'basic'` | How client credentials reach the token endpoint: `'basic'` or `'post'` |
|
|
167
|
+
|
|
168
|
+
#### Authenticating with OAuth 2.0
|
|
169
|
+
|
|
170
|
+
As an alternative to username and password, a connection can authenticate with an OAuth 2.0 access
|
|
171
|
+
token. The connector obtains a token using the client-credentials grant, caches it, and refreshes it
|
|
172
|
+
shortly before it expires.
|
|
173
|
+
|
|
174
|
+
Matching client-credentials configurations share one token cache and concurrent
|
|
175
|
+
refresh across sync and async connections, separate pools and threads in the same
|
|
176
|
+
process. Connection and transport ownership remain local. See the
|
|
177
|
+
[OAuth lifecycle](docs/OAUTH_LIFECYCLE.md) for matching settings and expiry behavior.
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
conn = Connection(
|
|
181
|
+
host=host,
|
|
182
|
+
port=443,
|
|
183
|
+
database=database,
|
|
184
|
+
cluster_name='<cluster-name>',
|
|
185
|
+
secure=True,
|
|
186
|
+
client_id='<client_id>',
|
|
187
|
+
client_secret='<client_secret>',
|
|
188
|
+
token_url='https://<your-workspace>/oauth2/token',
|
|
189
|
+
)
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
If you mint tokens yourself, pass one directly instead. The connector will not refresh it, so a
|
|
193
|
+
long-lived connection may outlive the token:
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
conn = Connection(
|
|
197
|
+
host=host, port=443, database=database, cluster_name='<cluster-name>',
|
|
198
|
+
secure=True, access_token='<token>',
|
|
199
|
+
)
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
**Supply exactly one authentication method.** Passing both username/password and OAuth settings
|
|
203
|
+
raises a `ValueError` rather than picking one, so a stale value left in a config file cannot quietly
|
|
204
|
+
win.
|
|
205
|
+
|
|
206
|
+
OAuth sends `authorization: Bearer <token>` on every RPC. It does not create an
|
|
207
|
+
authentication session; protobuf username, password and session ID fields remain
|
|
208
|
+
empty. The server stack must support and enable this bearer-only flow.
|
|
209
|
+
|
|
210
|
+
Use `secure=True` with your TLS endpoint and an HTTPS `token_url` to protect tokens
|
|
211
|
+
and client credentials. OAuth does not enable TLS automatically; `secure` defaults
|
|
212
|
+
to `False`. Set `cluster_name` to identify the target cluster explicitly.
|
|
213
|
+
|
|
214
|
+
With `auto_resume=True` (the default), OAuth can recover an initial query prepare
|
|
215
|
+
that returns the exact suspended-cluster response. This requires compatible,
|
|
216
|
+
enabled server components that authorize resume and report readiness. The connector
|
|
217
|
+
performs one recovery sequence and retries prepare after readiness. It never
|
|
218
|
+
replays query execution, metadata retrieval or result fetching, and it does not
|
|
219
|
+
treat generic connection failures as suspension.
|
|
220
|
+
|
|
221
|
+
Recovery has a shared 300-second default deadline covering lock waits, token
|
|
222
|
+
acquisition, status/resume calls, polling and the resumed prepare. Configure it with
|
|
223
|
+
`grpc_options={'grpc_auto_resume_timeout_seconds': 300}`. The deadline starts after
|
|
224
|
+
the suspended response; initial prepare has its own timeout. A dispatched resume
|
|
225
|
+
may still complete after the client times out.
|
|
226
|
+
|
|
227
|
+
Set `auto_resume=False` to disable recovery. Invalid credentials, denied permissions,
|
|
228
|
+
unsupported servers and failed or unknown cluster states stop recovery without
|
|
229
|
+
falling back to username/password authentication. Externally supplied access tokens
|
|
230
|
+
are not refreshed.
|
|
134
231
|
|
|
135
232
|
#### Secure Connection Example
|
|
136
233
|
|
|
@@ -372,6 +469,135 @@ database = '<new_database_name>' # Replace with the new database.
|
|
|
372
469
|
cursor = conn.cursor(database, catalog_name)
|
|
373
470
|
```
|
|
374
471
|
|
|
472
|
+
### Opt in to multi-chunk result batches
|
|
473
|
+
|
|
474
|
+
Set `enable_result_batch_v2=True` to enable V2 batches, continuous result downloading,
|
|
475
|
+
and bounded parallel chunk decoding together. It defaults to `False`. There
|
|
476
|
+
are no separate public prefetch or decoding switches. `arraysize` and
|
|
477
|
+
`fetchmany(size)` only change the number of rows returned to the application.
|
|
478
|
+
|
|
479
|
+
The planner must support V2 and have `ENABLE_GET_NEXT_RESULT_BATCH_V2` enabled.
|
|
480
|
+
`ENABLE_GET_NEXT_CHUNK_V2` controls a separate executor-to-planner boundary.
|
|
481
|
+
These flags do not extend the planner's query lifetime. Finishing a large result
|
|
482
|
+
within 900 seconds still needs a measurement on the target deployment.
|
|
483
|
+
|
|
484
|
+
The optimized path requires ordinary GIL-enabled CPython 3.11 through 3.13 and
|
|
485
|
+
an import-safe application entry point. Put application startup inside
|
|
486
|
+
`if __name__ == "__main__":`, including when using SQLAlchemy. Interactive,
|
|
487
|
+
daemon and frozen applications are rejected before query submission. Two
|
|
488
|
+
workers use the explicit `spawn` method; the application's global multiprocessing
|
|
489
|
+
start method is unchanged. An unguarded main script can run its other import
|
|
490
|
+
side effects, so keep all application startup inside the guard.
|
|
491
|
+
|
|
492
|
+
Both APIs default to unlimited gRPC send and receive message sizes (`-1`).
|
|
493
|
+
Explicit positive byte limits are still honored. For sync, set
|
|
494
|
+
`max_receive_message_length` and `max_send_message_length` in `grpc_options`.
|
|
495
|
+
For async, set the receive limit with `max_receive_message_bytes` or
|
|
496
|
+
`max_receive_message_length` in `grpc_options`; if both are supplied, they must
|
|
497
|
+
agree. Set `max_send_message_length` in `grpc_options` for the send limit.
|
|
498
|
+
The receive limit applies to each message after transport decompression, not the
|
|
499
|
+
total query result. Unlimited receiving does not limit client memory use;
|
|
500
|
+
continuous downloading can retain serialized batches while decoding catches up.
|
|
501
|
+
|
|
502
|
+
These examples use your existing `connection_options`, `sql`, and `consume`:
|
|
503
|
+
|
|
504
|
+
```python
|
|
505
|
+
from e6data_python_connector import Connection
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def main():
|
|
509
|
+
options = {**connection_options, "enable_result_batch_v2": True}
|
|
510
|
+
with Connection(**options) as connection:
|
|
511
|
+
with connection.cursor() as cursor:
|
|
512
|
+
cursor.execute(sql)
|
|
513
|
+
for rows in cursor.fetchall_buffer():
|
|
514
|
+
consume(rows)
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
if __name__ == "__main__":
|
|
518
|
+
main()
|
|
519
|
+
```
|
|
520
|
+
|
|
521
|
+
```python
|
|
522
|
+
import asyncio
|
|
523
|
+
from e6data_python_connector.aio import AsyncConnection
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
async def main():
|
|
527
|
+
options = {**connection_options, "enable_result_batch_v2": True}
|
|
528
|
+
async with AsyncConnection(**options) as connection:
|
|
529
|
+
async with connection.cursor() as cursor:
|
|
530
|
+
await cursor.execute(sql)
|
|
531
|
+
async for rows in cursor.fetchall_buffer():
|
|
532
|
+
consume(rows)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
if __name__ == "__main__":
|
|
536
|
+
asyncio.run(main())
|
|
537
|
+
```
|
|
538
|
+
|
|
539
|
+
SQLAlchemy accepts the same flag through `connect_args`. Create and use engines
|
|
540
|
+
inside the guarded application entry point:
|
|
541
|
+
|
|
542
|
+
```python
|
|
543
|
+
connect_args = {**connection_options, "enable_result_batch_v2": True}
|
|
544
|
+
engine = create_engine("e6data://", connect_args=connect_args)
|
|
545
|
+
async_engine = create_async_engine("e6data+asyncio://", connect_args=connect_args)
|
|
546
|
+
```
|
|
547
|
+
|
|
548
|
+
After the first V2 response, a download thread (sync) or task (async) keeps
|
|
549
|
+
fetching serialized batches until the engine returns end-of-stream. Decoding
|
|
550
|
+
runs concurrently and takes batches from the queue in order. Downloading does
|
|
551
|
+
not wait for decoding or application row processing. There is no queue-size
|
|
552
|
+
limit and no process-wide limit on active downloaders. Result RPCs for the same
|
|
553
|
+
query never overlap.
|
|
554
|
+
|
|
555
|
+
Each background RPC gets a fresh configured transport budget when it starts;
|
|
556
|
+
that deadline never extends after dispatch. An empty nonterminal response uses
|
|
557
|
+
backoff with a finite no-progress deadline. Completed responses can wait while
|
|
558
|
+
the application is idle. Public fetch deadlines still cover waiting and decoding,
|
|
559
|
+
and expiry or cancellation stops the downloader. Async `fetchall` keeps one
|
|
560
|
+
deadline for its entire operation.
|
|
561
|
+
|
|
562
|
+
Two shared worker processes can decode one multi-chunk envelope at a time.
|
|
563
|
+
Other envelopes use the sequential path. Workers receive chunk bytes and column
|
|
564
|
+
positions, not connections or credentials. Process startup and copying add cost,
|
|
565
|
+
so parallel decoding is not a promise of better performance for every result.
|
|
566
|
+
|
|
567
|
+
Fetch return shapes and row order stay the same. `fetchall_buffer()` yields the
|
|
568
|
+
original decoded chunks. All chunks in an envelope must decode successfully
|
|
569
|
+
before any are exposed. The serialized queue can hold the entire result when
|
|
570
|
+
downloading is faster than decoding or application reads. Workers add Python
|
|
571
|
+
objects and process copies; the receive limit is not a limit on total memory.
|
|
572
|
+
`fetchall()` still retains the full result.
|
|
573
|
+
|
|
574
|
+
If a decoder worker exits or its local pipe breaks, the connector stops the
|
|
575
|
+
failed workers and decodes that complete serialized envelope once in the parent
|
|
576
|
+
process. It uses the same deadline and makes no extra result RPC. Continuous
|
|
577
|
+
downloading stays active and uncapped during this recovery. Later envelopes use
|
|
578
|
+
sequential decoding until the decoder runtime is replaced through its normal
|
|
579
|
+
lifecycle. Cancellation, expired deadlines, invalid worker replies, and errors
|
|
580
|
+
reported while decoding data are still failures. Their original cause is kept
|
|
581
|
+
on the first error delivered to the caller.
|
|
582
|
+
|
|
583
|
+
Only a V2 `UNIMPLEMENTED` response switches that query to V1 and disables its
|
|
584
|
+
optimizations. Other RPC failures reach the caller as the original
|
|
585
|
+
`grpc.RpcError` or `grpc.aio.AioRpcError`. A failed pending RPC is raised when
|
|
586
|
+
its envelope is needed, after current valid chunks have been consumed. Failed
|
|
587
|
+
or cancelled queries are not replayed. Later fetches on an incomplete result
|
|
588
|
+
raise `IncompleteResultError`. Clear, cancel, close and pool return retire
|
|
589
|
+
pending work; unsafe connections must not be reused.
|
|
590
|
+
|
|
591
|
+
Use `debug=True` for payload-free RPC, decode and worker diagnostics. Serialized
|
|
592
|
+
Protobuf bytes are uncompressed payload size, not bytes sent over the network.
|
|
593
|
+
|
|
594
|
+
For rollback, close old cursors and connections, then omit
|
|
595
|
+
`enable_result_batch_v2` or set it to `False` on new connections. Closing the
|
|
596
|
+
last V2 connection also stops its shared workers. See the
|
|
597
|
+
[qualification instructions](test/README.md#result-batch-v2-qualification).
|
|
598
|
+
Local synthetic results do not prove customer performance or the 900-second
|
|
599
|
+
query requirement.
|
|
600
|
+
|
|
375
601
|
### Get Query Time Metrics
|
|
376
602
|
```python
|
|
377
603
|
import json
|
|
@@ -817,11 +1043,27 @@ conn = Connection(
|
|
|
817
1043
|
```
|
|
818
1044
|
|
|
819
1045
|
When `debug=True`, the following features are enabled:
|
|
820
|
-
-
|
|
1046
|
+
- Connector logging at DEBUG level, for both sync and async connections. Existing
|
|
1047
|
+
application logging handlers and the root logger level are kept.
|
|
821
1048
|
- Blue-green strategy transition logging
|
|
822
1049
|
- Connection lifecycle logging
|
|
823
1050
|
- Query execution detailed logging
|
|
824
1051
|
|
|
1052
|
+
Each result fetch writes a normal debug message with `protocol`, `rpc_seconds`,
|
|
1053
|
+
`decode_seconds`, `chunk_count`, `serialized_bytes`, and `status`. The timing values
|
|
1054
|
+
are seconds. `rpc_seconds` measures the client RPC call, including waiting for its
|
|
1055
|
+
response. `decode_seconds` measures client response processing. `serialized_bytes`
|
|
1056
|
+
is the protobuf response size, not the number of bytes on the network. A failed
|
|
1057
|
+
attempt reports zero response size and chunk count. The same values remain available
|
|
1058
|
+
as `result_batch_*` attributes for structured log handlers.
|
|
1059
|
+
|
|
1060
|
+
These metrics do not add SQL, credentials, session IDs, result values, or raw metadata
|
|
1061
|
+
to logs. `debug=False` keeps the normal quiet default. Applications can also enable
|
|
1062
|
+
DEBUG on the `e6data_python_connector` logger themselves. `debug=True` adds a connector
|
|
1063
|
+
stream handler only when no existing handler in its logger path accepts DEBUG.
|
|
1064
|
+
As with normal Python logging, enabling the package logger also affects other open
|
|
1065
|
+
connector connections in that process.
|
|
1066
|
+
|
|
825
1067
|
### gRPC Network Tracing
|
|
826
1068
|
|
|
827
1069
|
For low-level gRPC network debugging (HTTP/2 frames, TCP events), set environment variables **before** running your Python script:
|
|
@@ -1,10 +1,30 @@
|
|
|
1
1
|
# e6data Python Connector
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Package version and extras are defined in `setup.py`.
|
|
4
4
|
|
|
5
5
|
## Introduction
|
|
6
6
|
|
|
7
|
-
The e6data Connector for Python provides an interface for writing Python applications that can connect to e6data and perform operations. It includes
|
|
7
|
+
The e6data Connector for Python provides an interface for writing Python applications that can connect to e6data and perform operations. It includes routing support for blue-green deployments. Active query handles remain tied to the engine that accepted them; failover does not guarantee uninterrupted results.
|
|
8
|
+
|
|
9
|
+
## Optional native asyncio API
|
|
10
|
+
|
|
11
|
+
This branch adds native async connection, cursor, pooling and SQLAlchemy support for Python 3.11+. Release qualification remains incomplete. Install this branch's built package with `[async]` or `[async-sqlalchemy]`; these instructions do not assert that the feature is published.
|
|
12
|
+
|
|
13
|
+
```python
|
|
14
|
+
from e6data_python_connector.aio import AsyncConnection
|
|
15
|
+
|
|
16
|
+
async def query(config, sql):
|
|
17
|
+
async with AsyncConnection(**config) as connection:
|
|
18
|
+
async with connection.cursor() as cursor:
|
|
19
|
+
await cursor.execute(sql)
|
|
20
|
+
return await cursor.fetchall()
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
`config` is application-supplied connection configuration. OAuth requires verified TLS. Native `fetchone()` preserves the one-row outer list, while SQLAlchemy `e6data+asyncio` adapts it to a normal row. Ambiguous submission and incomplete result errors must not be handled by blindly retrying the query. Use context managers for bounded cleanup.
|
|
24
|
+
|
|
25
|
+
See [Async API](docs/ASYNC_API.md) for the complete API map, deadlines, pooling, SQLAlchemy and test configuration, [OAuth lifecycle](docs/OAUTH_LIFECYCLE.md) for renewal and failure behavior, and [the application example](examples/async_query.py).
|
|
26
|
+
|
|
27
|
+
The new CI keeps the full-package coverage denominator and a greater-than-80% gate. The initial baseline was about 19%, so focused async test success is not a release pass. Real-service tests require explicit configuration and are qualified separately.
|
|
8
28
|
|
|
9
29
|
### Dependencies
|
|
10
30
|
Make sure to install below dependencies and wheel before install e6data-python-connector.
|
|
@@ -81,8 +101,8 @@ The `Connection` class supports the following parameters:
|
|
|
81
101
|
|-----------|------|----------|---------|-------------|
|
|
82
102
|
| `host` | str | Yes | - | IP address or hostname of the e6data cluster |
|
|
83
103
|
| `port` | int | Yes | - | Port of the e6data engine (typically 80) |
|
|
84
|
-
| `username` | str |
|
|
85
|
-
| `password` | str |
|
|
104
|
+
| `username` | str | Conditional | None | Your e6data Email ID. Required unless authenticating with OAuth |
|
|
105
|
+
| `password` | str | Conditional | None | Access Token generated in the e6data console. Required unless authenticating with OAuth |
|
|
86
106
|
| `database` | str | No | None | Database to perform queries on |
|
|
87
107
|
| `catalog` | str | No | None | Catalog name |
|
|
88
108
|
| `cluster_name` | str | No | None | Name of the cluster for cluster-specific operations |
|
|
@@ -92,6 +112,76 @@ The `Connection` class supports the following parameters:
|
|
|
92
112
|
| `grpc_options` | dict | No | None | Additional gRPC configuration options |
|
|
93
113
|
| `debug` | bool | No | False | Enable debug logging for troubleshooting |
|
|
94
114
|
| `require_fastbinary` | bool | No | True | Require fastbinary module for Thrift deserialization. Set to False to use pure Python implementation if system dependencies cannot be installed |
|
|
115
|
+
| `client_id` | str | No | None | OAuth 2.0 client id. Use with `client_secret` and `token_url` |
|
|
116
|
+
| `client_secret` | str | No | None | OAuth 2.0 client secret |
|
|
117
|
+
| `token_url` | str | No | None | Token endpoint of the authorization server |
|
|
118
|
+
| `oauth_scope` | str | No | None | Space-delimited scopes to request. Omit for the client's full registered set |
|
|
119
|
+
| `access_token` | str | No | None | A previously obtained access token, for callers that mint their own |
|
|
120
|
+
| `client_auth_method` | str | No | `'basic'` | How client credentials reach the token endpoint: `'basic'` or `'post'` |
|
|
121
|
+
|
|
122
|
+
#### Authenticating with OAuth 2.0
|
|
123
|
+
|
|
124
|
+
As an alternative to username and password, a connection can authenticate with an OAuth 2.0 access
|
|
125
|
+
token. The connector obtains a token using the client-credentials grant, caches it, and refreshes it
|
|
126
|
+
shortly before it expires.
|
|
127
|
+
|
|
128
|
+
Matching client-credentials configurations share one token cache and concurrent
|
|
129
|
+
refresh across sync and async connections, separate pools and threads in the same
|
|
130
|
+
process. Connection and transport ownership remain local. See the
|
|
131
|
+
[OAuth lifecycle](docs/OAUTH_LIFECYCLE.md) for matching settings and expiry behavior.
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
conn = Connection(
|
|
135
|
+
host=host,
|
|
136
|
+
port=443,
|
|
137
|
+
database=database,
|
|
138
|
+
cluster_name='<cluster-name>',
|
|
139
|
+
secure=True,
|
|
140
|
+
client_id='<client_id>',
|
|
141
|
+
client_secret='<client_secret>',
|
|
142
|
+
token_url='https://<your-workspace>/oauth2/token',
|
|
143
|
+
)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
If you mint tokens yourself, pass one directly instead. The connector will not refresh it, so a
|
|
147
|
+
long-lived connection may outlive the token:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
conn = Connection(
|
|
151
|
+
host=host, port=443, database=database, cluster_name='<cluster-name>',
|
|
152
|
+
secure=True, access_token='<token>',
|
|
153
|
+
)
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
**Supply exactly one authentication method.** Passing both username/password and OAuth settings
|
|
157
|
+
raises a `ValueError` rather than picking one, so a stale value left in a config file cannot quietly
|
|
158
|
+
win.
|
|
159
|
+
|
|
160
|
+
OAuth sends `authorization: Bearer <token>` on every RPC. It does not create an
|
|
161
|
+
authentication session; protobuf username, password and session ID fields remain
|
|
162
|
+
empty. The server stack must support and enable this bearer-only flow.
|
|
163
|
+
|
|
164
|
+
Use `secure=True` with your TLS endpoint and an HTTPS `token_url` to protect tokens
|
|
165
|
+
and client credentials. OAuth does not enable TLS automatically; `secure` defaults
|
|
166
|
+
to `False`. Set `cluster_name` to identify the target cluster explicitly.
|
|
167
|
+
|
|
168
|
+
With `auto_resume=True` (the default), OAuth can recover an initial query prepare
|
|
169
|
+
that returns the exact suspended-cluster response. This requires compatible,
|
|
170
|
+
enabled server components that authorize resume and report readiness. The connector
|
|
171
|
+
performs one recovery sequence and retries prepare after readiness. It never
|
|
172
|
+
replays query execution, metadata retrieval or result fetching, and it does not
|
|
173
|
+
treat generic connection failures as suspension.
|
|
174
|
+
|
|
175
|
+
Recovery has a shared 300-second default deadline covering lock waits, token
|
|
176
|
+
acquisition, status/resume calls, polling and the resumed prepare. Configure it with
|
|
177
|
+
`grpc_options={'grpc_auto_resume_timeout_seconds': 300}`. The deadline starts after
|
|
178
|
+
the suspended response; initial prepare has its own timeout. A dispatched resume
|
|
179
|
+
may still complete after the client times out.
|
|
180
|
+
|
|
181
|
+
Set `auto_resume=False` to disable recovery. Invalid credentials, denied permissions,
|
|
182
|
+
unsupported servers and failed or unknown cluster states stop recovery without
|
|
183
|
+
falling back to username/password authentication. Externally supplied access tokens
|
|
184
|
+
are not refreshed.
|
|
95
185
|
|
|
96
186
|
#### Secure Connection Example
|
|
97
187
|
|
|
@@ -333,6 +423,135 @@ database = '<new_database_name>' # Replace with the new database.
|
|
|
333
423
|
cursor = conn.cursor(database, catalog_name)
|
|
334
424
|
```
|
|
335
425
|
|
|
426
|
+
### Opt in to multi-chunk result batches
|
|
427
|
+
|
|
428
|
+
Set `enable_result_batch_v2=True` to enable V2 batches, continuous result downloading,
|
|
429
|
+
and bounded parallel chunk decoding together. It defaults to `False`. There
|
|
430
|
+
are no separate public prefetch or decoding switches. `arraysize` and
|
|
431
|
+
`fetchmany(size)` only change the number of rows returned to the application.
|
|
432
|
+
|
|
433
|
+
The planner must support V2 and have `ENABLE_GET_NEXT_RESULT_BATCH_V2` enabled.
|
|
434
|
+
`ENABLE_GET_NEXT_CHUNK_V2` controls a separate executor-to-planner boundary.
|
|
435
|
+
These flags do not extend the planner's query lifetime. Finishing a large result
|
|
436
|
+
within 900 seconds still needs a measurement on the target deployment.
|
|
437
|
+
|
|
438
|
+
The optimized path requires ordinary GIL-enabled CPython 3.11 through 3.13 and
|
|
439
|
+
an import-safe application entry point. Put application startup inside
|
|
440
|
+
`if __name__ == "__main__":`, including when using SQLAlchemy. Interactive,
|
|
441
|
+
daemon and frozen applications are rejected before query submission. Two
|
|
442
|
+
workers use the explicit `spawn` method; the application's global multiprocessing
|
|
443
|
+
start method is unchanged. An unguarded main script can run its other import
|
|
444
|
+
side effects, so keep all application startup inside the guard.
|
|
445
|
+
|
|
446
|
+
Both APIs default to unlimited gRPC send and receive message sizes (`-1`).
|
|
447
|
+
Explicit positive byte limits are still honored. For sync, set
|
|
448
|
+
`max_receive_message_length` and `max_send_message_length` in `grpc_options`.
|
|
449
|
+
For async, set the receive limit with `max_receive_message_bytes` or
|
|
450
|
+
`max_receive_message_length` in `grpc_options`; if both are supplied, they must
|
|
451
|
+
agree. Set `max_send_message_length` in `grpc_options` for the send limit.
|
|
452
|
+
The receive limit applies to each message after transport decompression, not the
|
|
453
|
+
total query result. Unlimited receiving does not limit client memory use;
|
|
454
|
+
continuous downloading can retain serialized batches while decoding catches up.
|
|
455
|
+
|
|
456
|
+
These examples use your existing `connection_options`, `sql`, and `consume`:
|
|
457
|
+
|
|
458
|
+
```python
|
|
459
|
+
from e6data_python_connector import Connection
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def main():
|
|
463
|
+
options = {**connection_options, "enable_result_batch_v2": True}
|
|
464
|
+
with Connection(**options) as connection:
|
|
465
|
+
with connection.cursor() as cursor:
|
|
466
|
+
cursor.execute(sql)
|
|
467
|
+
for rows in cursor.fetchall_buffer():
|
|
468
|
+
consume(rows)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
if __name__ == "__main__":
|
|
472
|
+
main()
|
|
473
|
+
```
|
|
474
|
+
|
|
475
|
+
```python
|
|
476
|
+
import asyncio
|
|
477
|
+
from e6data_python_connector.aio import AsyncConnection
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
async def main():
|
|
481
|
+
options = {**connection_options, "enable_result_batch_v2": True}
|
|
482
|
+
async with AsyncConnection(**options) as connection:
|
|
483
|
+
async with connection.cursor() as cursor:
|
|
484
|
+
await cursor.execute(sql)
|
|
485
|
+
async for rows in cursor.fetchall_buffer():
|
|
486
|
+
consume(rows)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
if __name__ == "__main__":
|
|
490
|
+
asyncio.run(main())
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
SQLAlchemy accepts the same flag through `connect_args`. Create and use engines
|
|
494
|
+
inside the guarded application entry point:
|
|
495
|
+
|
|
496
|
+
```python
|
|
497
|
+
connect_args = {**connection_options, "enable_result_batch_v2": True}
|
|
498
|
+
engine = create_engine("e6data://", connect_args=connect_args)
|
|
499
|
+
async_engine = create_async_engine("e6data+asyncio://", connect_args=connect_args)
|
|
500
|
+
```
|
|
501
|
+
|
|
502
|
+
After the first V2 response, a download thread (sync) or task (async) keeps
|
|
503
|
+
fetching serialized batches until the engine returns end-of-stream. Decoding
|
|
504
|
+
runs concurrently and takes batches from the queue in order. Downloading does
|
|
505
|
+
not wait for decoding or application row processing. There is no queue-size
|
|
506
|
+
limit and no process-wide limit on active downloaders. Result RPCs for the same
|
|
507
|
+
query never overlap.
|
|
508
|
+
|
|
509
|
+
Each background RPC gets a fresh configured transport budget when it starts;
|
|
510
|
+
that deadline never extends after dispatch. An empty nonterminal response uses
|
|
511
|
+
backoff with a finite no-progress deadline. Completed responses can wait while
|
|
512
|
+
the application is idle. Public fetch deadlines still cover waiting and decoding,
|
|
513
|
+
and expiry or cancellation stops the downloader. Async `fetchall` keeps one
|
|
514
|
+
deadline for its entire operation.
|
|
515
|
+
|
|
516
|
+
Two shared worker processes can decode one multi-chunk envelope at a time.
|
|
517
|
+
Other envelopes use the sequential path. Workers receive chunk bytes and column
|
|
518
|
+
positions, not connections or credentials. Process startup and copying add cost,
|
|
519
|
+
so parallel decoding is not a promise of better performance for every result.
|
|
520
|
+
|
|
521
|
+
Fetch return shapes and row order stay the same. `fetchall_buffer()` yields the
|
|
522
|
+
original decoded chunks. All chunks in an envelope must decode successfully
|
|
523
|
+
before any are exposed. The serialized queue can hold the entire result when
|
|
524
|
+
downloading is faster than decoding or application reads. Workers add Python
|
|
525
|
+
objects and process copies; the receive limit is not a limit on total memory.
|
|
526
|
+
`fetchall()` still retains the full result.
|
|
527
|
+
|
|
528
|
+
If a decoder worker exits or its local pipe breaks, the connector stops the
|
|
529
|
+
failed workers and decodes that complete serialized envelope once in the parent
|
|
530
|
+
process. It uses the same deadline and makes no extra result RPC. Continuous
|
|
531
|
+
downloading stays active and uncapped during this recovery. Later envelopes use
|
|
532
|
+
sequential decoding until the decoder runtime is replaced through its normal
|
|
533
|
+
lifecycle. Cancellation, expired deadlines, invalid worker replies, and errors
|
|
534
|
+
reported while decoding data are still failures. Their original cause is kept
|
|
535
|
+
on the first error delivered to the caller.
|
|
536
|
+
|
|
537
|
+
Only a V2 `UNIMPLEMENTED` response switches that query to V1 and disables its
|
|
538
|
+
optimizations. Other RPC failures reach the caller as the original
|
|
539
|
+
`grpc.RpcError` or `grpc.aio.AioRpcError`. A failed pending RPC is raised when
|
|
540
|
+
its envelope is needed, after current valid chunks have been consumed. Failed
|
|
541
|
+
or cancelled queries are not replayed. Later fetches on an incomplete result
|
|
542
|
+
raise `IncompleteResultError`. Clear, cancel, close and pool return retire
|
|
543
|
+
pending work; unsafe connections must not be reused.
|
|
544
|
+
|
|
545
|
+
Use `debug=True` for payload-free RPC, decode and worker diagnostics. Serialized
|
|
546
|
+
Protobuf bytes are uncompressed payload size, not bytes sent over the network.
|
|
547
|
+
|
|
548
|
+
For rollback, close old cursors and connections, then omit
|
|
549
|
+
`enable_result_batch_v2` or set it to `False` on new connections. Closing the
|
|
550
|
+
last V2 connection also stops its shared workers. See the
|
|
551
|
+
[qualification instructions](test/README.md#result-batch-v2-qualification).
|
|
552
|
+
Local synthetic results do not prove customer performance or the 900-second
|
|
553
|
+
query requirement.
|
|
554
|
+
|
|
336
555
|
### Get Query Time Metrics
|
|
337
556
|
```python
|
|
338
557
|
import json
|
|
@@ -778,11 +997,27 @@ conn = Connection(
|
|
|
778
997
|
```
|
|
779
998
|
|
|
780
999
|
When `debug=True`, the following features are enabled:
|
|
781
|
-
-
|
|
1000
|
+
- Connector logging at DEBUG level, for both sync and async connections. Existing
|
|
1001
|
+
application logging handlers and the root logger level are kept.
|
|
782
1002
|
- Blue-green strategy transition logging
|
|
783
1003
|
- Connection lifecycle logging
|
|
784
1004
|
- Query execution detailed logging
|
|
785
1005
|
|
|
1006
|
+
Each result fetch writes a normal debug message with `protocol`, `rpc_seconds`,
|
|
1007
|
+
`decode_seconds`, `chunk_count`, `serialized_bytes`, and `status`. The timing values
|
|
1008
|
+
are seconds. `rpc_seconds` measures the client RPC call, including waiting for its
|
|
1009
|
+
response. `decode_seconds` measures client response processing. `serialized_bytes`
|
|
1010
|
+
is the protobuf response size, not the number of bytes on the network. A failed
|
|
1011
|
+
attempt reports zero response size and chunk count. The same values remain available
|
|
1012
|
+
as `result_batch_*` attributes for structured log handlers.
|
|
1013
|
+
|
|
1014
|
+
These metrics do not add SQL, credentials, session IDs, result values, or raw metadata
|
|
1015
|
+
to logs. `debug=False` keeps the normal quiet default. Applications can also enable
|
|
1016
|
+
DEBUG on the `e6data_python_connector` logger themselves. `debug=True` adds a connector
|
|
1017
|
+
stream handler only when no existing handler in its logger path accepts DEBUG.
|
|
1018
|
+
As with normal Python logging, enabling the package logger also affects other open
|
|
1019
|
+
connector connections in that process.
|
|
1020
|
+
|
|
786
1021
|
### gRPC Network Tracing
|
|
787
1022
|
|
|
788
1023
|
For low-level gRPC network debugging (HTTP/2 frames, TCP events), set environment variables **before** running your Python script:
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Explicit optional native asyncio entry point (Python 3.11+)."""
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
if sys.version_info < (3, 11):
|
|
5
|
+
raise RuntimeError('The async API requires Python 3.11 or newer.')
|
|
6
|
+
try:
|
|
7
|
+
import httpx as _httpx
|
|
8
|
+
except ImportError:
|
|
9
|
+
raise ImportError('Install e6data-python-connector[async] for the async API.') from None
|
|
10
|
+
|
|
11
|
+
from .async_connection import AsyncConnection
|
|
12
|
+
|
|
13
|
+
__all__ = ['connect', 'AsyncConnection', 'AsyncCursor', 'AsyncConnectionPool', 'AsyncClusterManager']
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
async def connect(*args, **kwargs):
|
|
17
|
+
return await AsyncConnection(*args, **kwargs).open()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def __getattr__(name):
|
|
21
|
+
modules = {'AsyncCursor': 'async_cursor', 'AsyncConnectionPool': 'async_connection_pool',
|
|
22
|
+
'AsyncClusterManager': 'async_cluster_manager'}
|
|
23
|
+
if name not in modules:
|
|
24
|
+
raise AttributeError(name)
|
|
25
|
+
from importlib import import_module
|
|
26
|
+
return getattr(import_module('.' + modules[name], __package__), name)
|