e6data-python-connector 2.3.12rc12__tar.gz → 2.3.12rc31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {e6data_python_connector-2.3.12rc12/e6data_python_connector.egg-info → e6data_python_connector-2.3.12rc31}/PKG-INFO +32 -61
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/README.md +31 -60
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/cluster_manager.py +2 -76
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/datainputstream.py +24 -7
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/e6data_grpc.py +35 -76
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31/e6data_python_connector.egg-info}/PKG-INFO +32 -61
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/setup.py +1 -1
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/LICENSE +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/MANIFEST.in +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/cluster_server/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/cluster_server/cluster_pb2.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/cluster_server/cluster_pb2_grpc.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/common.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/connection_pool.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/constants.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/date_time_utils.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/dialect.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/e6x_vector/constants.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/e6x_vector/ttypes.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/exceptions.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/server/QueryEngineService.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/server/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/server/constants.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/server/e6x_engine_pb2.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/server/e6x_engine_pb2.pyi +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/server/e6x_engine_pb2_grpc.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/server/ttypes.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/strategy.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector/typeId.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector.egg-info/SOURCES.txt +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector.egg-info/dependency_links.txt +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector.egg-info/entry_points.txt +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector.egg-info/requires.txt +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/e6data_python_connector.egg-info/top_level.txt +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/gen-py/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/gen-py/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/gen-py/e6x_vector/constants.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/gen-py/e6x_vector/ttypes.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/pyproject.toml +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/setup.cfg +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_38_nines.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_all_cases.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_binary.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_correct_value.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_fields.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/check_decimal_errors.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/cleanup_test_files.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/debug_38_nines.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/debug_binary.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/final_test.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/move_tests.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/quick_test.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_38_nines.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_all_decimal128_cases.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_cluster_manager_efficiency.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_cluster_manager_none_strategy.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_cluster_manager_strategy.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_comprehensive.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_connection_pool.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_connection_pool_e2e.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_current_implementation.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_decimal128_binary_parsing.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_decimal128_parsing.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_fix.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_improved_parsing.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_known_case.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_manual_analysis.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_mock_server.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_multiprocessing_fix.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_new_implementation.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_pool_concurrency_simple.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_pool_threading_multiprocessing.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_simple_connection_pool.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_specific_binary.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_strategy.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_strategy_logic.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_strategy_persistence_fix.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_strategy_sharing_fix.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_user_binary_value.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/tests.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/tests_grpc.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/validate_decimal128.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/validate_implementation.py +0 -0
- {e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/verify_decimal_fix.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.12rc31
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
6
|
Author: e6data, Inc.
|
|
@@ -39,7 +39,7 @@ Dynamic: summary
|
|
|
39
39
|
|
|
40
40
|
# e6data Python Connector
|
|
41
41
|
|
|
42
|
-

|
|
43
43
|
|
|
44
44
|
## Introduction
|
|
45
45
|
|
|
@@ -125,7 +125,7 @@ The `Connection` class supports the following parameters:
|
|
|
125
125
|
| `auto_resume` | bool | No | True | Automatically resume cluster if suspended |
|
|
126
126
|
| `grpc_options` | dict | No | None | Additional gRPC configuration options |
|
|
127
127
|
| `debug` | bool | No | False | Enable debug logging for troubleshooting |
|
|
128
|
-
| `
|
|
128
|
+
| `require_fastbinary` | bool | No | True | Require fastbinary module for Thrift deserialization. Set to False to use pure Python implementation if system dependencies cannot be installed |
|
|
129
129
|
|
|
130
130
|
#### Secure Connection Example
|
|
131
131
|
|
|
@@ -143,14 +143,11 @@ conn = Connection(
|
|
|
143
143
|
)
|
|
144
144
|
```
|
|
145
145
|
|
|
146
|
-
####
|
|
146
|
+
#### Cluster-Specific Connection
|
|
147
147
|
|
|
148
|
-
When
|
|
148
|
+
When working with multiple clusters, specify the cluster name:
|
|
149
149
|
|
|
150
150
|
```python
|
|
151
|
-
from e6data_python_connector import Connection
|
|
152
|
-
|
|
153
|
-
# Option 1: Using CA certificate file path
|
|
154
151
|
conn = Connection(
|
|
155
152
|
host=host,
|
|
156
153
|
port=port,
|
|
@@ -158,40 +155,30 @@ conn = Connection(
|
|
|
158
155
|
password=password,
|
|
159
156
|
database=database,
|
|
160
157
|
cluster_name='analytics-cluster-01', # Specify cluster name
|
|
161
|
-
secure=True
|
|
162
|
-
ssl_cert='/path/to/ca-cert.pem' # Path to CA certificate
|
|
158
|
+
secure=True
|
|
163
159
|
)
|
|
160
|
+
```
|
|
164
161
|
|
|
165
|
-
|
|
166
|
-
with open('/path/to/ca-cert.pem', 'rb') as f:
|
|
167
|
-
ca_cert = f.read()
|
|
162
|
+
#### Handling Missing System Dependencies
|
|
168
163
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
password=password,
|
|
174
|
-
database=database,
|
|
175
|
-
secure=True,
|
|
176
|
-
ssl_cert=ca_cert # Certificate as bytes
|
|
177
|
-
)
|
|
164
|
+
The e6data connector uses the `fastbinary` module (from Apache Thrift) for optimal performance when deserializing data. This module requires system-level dependencies (`python3-devel` and `gcc-c++`) to be installed.
|
|
165
|
+
|
|
166
|
+
**Default Behavior (Recommended):**
|
|
167
|
+
By default, the connector requires `fastbinary` to be available. If it's not found, the connection will fail immediately with a clear error message:
|
|
178
168
|
|
|
179
|
-
|
|
169
|
+
```python
|
|
180
170
|
conn = Connection(
|
|
181
171
|
host=host,
|
|
182
172
|
port=port,
|
|
183
173
|
username=username,
|
|
184
174
|
password=password,
|
|
185
|
-
database=database
|
|
186
|
-
secure=True # Uses system CA bundle
|
|
175
|
+
database=database
|
|
187
176
|
)
|
|
177
|
+
# Raises exception if fastbinary is not available
|
|
188
178
|
```
|
|
189
179
|
|
|
190
|
-
**
|
|
191
|
-
|
|
192
|
-
#### Cluster-Specific Connection
|
|
193
|
-
|
|
194
|
-
When working with multiple clusters, specify the cluster name:
|
|
180
|
+
**Fallback to Pure Python:**
|
|
181
|
+
If you cannot install system dependencies (e.g., in restricted environments, serverless platforms, or containers without build tools), you can disable the `fastbinary` requirement. The connector will fall back to a pure Python implementation with a performance penalty:
|
|
195
182
|
|
|
196
183
|
```python
|
|
197
184
|
conn = Connection(
|
|
@@ -200,11 +187,23 @@ conn = Connection(
|
|
|
200
187
|
username=username,
|
|
201
188
|
password=password,
|
|
202
189
|
database=database,
|
|
203
|
-
|
|
204
|
-
secure=True
|
|
190
|
+
require_fastbinary=False # Allow operation without fastbinary
|
|
205
191
|
)
|
|
192
|
+
# Logs warning but continues with pure Python implementation
|
|
206
193
|
```
|
|
207
194
|
|
|
195
|
+
**When to use `require_fastbinary=False`:**
|
|
196
|
+
- Running in AWS Lambda or other serverless environments
|
|
197
|
+
- Docker containers built without compilation tools
|
|
198
|
+
- Restricted environments where system packages cannot be installed
|
|
199
|
+
- Development/testing environments where performance is not critical
|
|
200
|
+
|
|
201
|
+
**Performance Impact:**
|
|
202
|
+
- With `fastbinary`: Optimal performance for data deserialization
|
|
203
|
+
- Without `fastbinary` (pure Python): ~2-3x slower deserialization, but otherwise fully functional
|
|
204
|
+
|
|
205
|
+
**Note:** It's strongly recommended to install system dependencies when possible for best performance. The `require_fastbinary=False` option should only be used when system dependencies cannot be installed.
|
|
206
|
+
|
|
208
207
|
### Perform a Queries & Get Results
|
|
209
208
|
|
|
210
209
|
```python
|
|
@@ -551,35 +550,6 @@ pool.return_connection(conn)
|
|
|
551
550
|
pool.close_all()
|
|
552
551
|
```
|
|
553
552
|
|
|
554
|
-
##### Connection Pool with HTTPS (HAProxy)
|
|
555
|
-
|
|
556
|
-
```python
|
|
557
|
-
from e6data_python_connector import ConnectionPool
|
|
558
|
-
|
|
559
|
-
# Connection pool with HTTPS and custom CA certificate
|
|
560
|
-
pool = ConnectionPool(
|
|
561
|
-
min_size=2,
|
|
562
|
-
max_size=10,
|
|
563
|
-
host=host,
|
|
564
|
-
port=port,
|
|
565
|
-
username=username,
|
|
566
|
-
password=password,
|
|
567
|
-
database=database,
|
|
568
|
-
secure=True,
|
|
569
|
-
ssl_cert='/path/to/ca-cert.pem' # CA certificate for HTTPS
|
|
570
|
-
)
|
|
571
|
-
|
|
572
|
-
# Use the pool with context manager for automatic cleanup
|
|
573
|
-
with pool.get_connection_context() as conn:
|
|
574
|
-
cursor = conn.cursor()
|
|
575
|
-
cursor.execute("SELECT * FROM table")
|
|
576
|
-
results = cursor.fetchall()
|
|
577
|
-
print(f"Retrieved {len(results)} rows")
|
|
578
|
-
|
|
579
|
-
# Clean up pool when done
|
|
580
|
-
pool.close_all()
|
|
581
|
-
```
|
|
582
|
-
|
|
583
553
|
##### Using Context Manager (Recommended)
|
|
584
554
|
|
|
585
555
|
The context manager pattern ensures connections are automatically returned to the pool:
|
|
@@ -767,5 +737,6 @@ python your_script.py
|
|
|
767
737
|
| 456 Strategy Error | Automatic blue-green failover will handle this |
|
|
768
738
|
| Memory issues with large results | Use `fetchall_buffer()` instead of `fetchall()` |
|
|
769
739
|
| gRPC message size errors | Configure `grpc_options` with appropriate message size limits |
|
|
740
|
+
| fastbinary import error | Install system dependencies (`python3-devel`, `gcc-c++`) or set `require_fastbinary=False` |
|
|
770
741
|
|
|
771
742
|
See [TECH_DOC.md](TECH_DOC.md) for detailed technical documentation.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# e6data Python Connector
|
|
2
2
|
|
|
3
|
-

|
|
4
4
|
|
|
5
5
|
## Introduction
|
|
6
6
|
|
|
@@ -86,7 +86,7 @@ The `Connection` class supports the following parameters:
|
|
|
86
86
|
| `auto_resume` | bool | No | True | Automatically resume cluster if suspended |
|
|
87
87
|
| `grpc_options` | dict | No | None | Additional gRPC configuration options |
|
|
88
88
|
| `debug` | bool | No | False | Enable debug logging for troubleshooting |
|
|
89
|
-
| `
|
|
89
|
+
| `require_fastbinary` | bool | No | True | Require fastbinary module for Thrift deserialization. Set to False to use pure Python implementation if system dependencies cannot be installed |
|
|
90
90
|
|
|
91
91
|
#### Secure Connection Example
|
|
92
92
|
|
|
@@ -104,14 +104,11 @@ conn = Connection(
|
|
|
104
104
|
)
|
|
105
105
|
```
|
|
106
106
|
|
|
107
|
-
####
|
|
107
|
+
#### Cluster-Specific Connection
|
|
108
108
|
|
|
109
|
-
When
|
|
109
|
+
When working with multiple clusters, specify the cluster name:
|
|
110
110
|
|
|
111
111
|
```python
|
|
112
|
-
from e6data_python_connector import Connection
|
|
113
|
-
|
|
114
|
-
# Option 1: Using CA certificate file path
|
|
115
112
|
conn = Connection(
|
|
116
113
|
host=host,
|
|
117
114
|
port=port,
|
|
@@ -119,40 +116,30 @@ conn = Connection(
|
|
|
119
116
|
password=password,
|
|
120
117
|
database=database,
|
|
121
118
|
cluster_name='analytics-cluster-01', # Specify cluster name
|
|
122
|
-
secure=True
|
|
123
|
-
ssl_cert='/path/to/ca-cert.pem' # Path to CA certificate
|
|
119
|
+
secure=True
|
|
124
120
|
)
|
|
121
|
+
```
|
|
125
122
|
|
|
126
|
-
|
|
127
|
-
with open('/path/to/ca-cert.pem', 'rb') as f:
|
|
128
|
-
ca_cert = f.read()
|
|
123
|
+
#### Handling Missing System Dependencies
|
|
129
124
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
password=password,
|
|
135
|
-
database=database,
|
|
136
|
-
secure=True,
|
|
137
|
-
ssl_cert=ca_cert # Certificate as bytes
|
|
138
|
-
)
|
|
125
|
+
The e6data connector uses the `fastbinary` module (from Apache Thrift) for optimal performance when deserializing data. This module requires system-level dependencies (`python3-devel` and `gcc-c++`) to be installed.
|
|
126
|
+
|
|
127
|
+
**Default Behavior (Recommended):**
|
|
128
|
+
By default, the connector requires `fastbinary` to be available. If it's not found, the connection will fail immediately with a clear error message:
|
|
139
129
|
|
|
140
|
-
|
|
130
|
+
```python
|
|
141
131
|
conn = Connection(
|
|
142
132
|
host=host,
|
|
143
133
|
port=port,
|
|
144
134
|
username=username,
|
|
145
135
|
password=password,
|
|
146
|
-
database=database
|
|
147
|
-
secure=True # Uses system CA bundle
|
|
136
|
+
database=database
|
|
148
137
|
)
|
|
138
|
+
# Raises exception if fastbinary is not available
|
|
149
139
|
```
|
|
150
140
|
|
|
151
|
-
**
|
|
152
|
-
|
|
153
|
-
#### Cluster-Specific Connection
|
|
154
|
-
|
|
155
|
-
When working with multiple clusters, specify the cluster name:
|
|
141
|
+
**Fallback to Pure Python:**
|
|
142
|
+
If you cannot install system dependencies (e.g., in restricted environments, serverless platforms, or containers without build tools), you can disable the `fastbinary` requirement. The connector will fall back to a pure Python implementation with a performance penalty:
|
|
156
143
|
|
|
157
144
|
```python
|
|
158
145
|
conn = Connection(
|
|
@@ -161,11 +148,23 @@ conn = Connection(
|
|
|
161
148
|
username=username,
|
|
162
149
|
password=password,
|
|
163
150
|
database=database,
|
|
164
|
-
|
|
165
|
-
secure=True
|
|
151
|
+
require_fastbinary=False # Allow operation without fastbinary
|
|
166
152
|
)
|
|
153
|
+
# Logs warning but continues with pure Python implementation
|
|
167
154
|
```
|
|
168
155
|
|
|
156
|
+
**When to use `require_fastbinary=False`:**
|
|
157
|
+
- Running in AWS Lambda or other serverless environments
|
|
158
|
+
- Docker containers built without compilation tools
|
|
159
|
+
- Restricted environments where system packages cannot be installed
|
|
160
|
+
- Development/testing environments where performance is not critical
|
|
161
|
+
|
|
162
|
+
**Performance Impact:**
|
|
163
|
+
- With `fastbinary`: Optimal performance for data deserialization
|
|
164
|
+
- Without `fastbinary` (pure Python): ~2-3x slower deserialization, but otherwise fully functional
|
|
165
|
+
|
|
166
|
+
**Note:** It's strongly recommended to install system dependencies when possible for best performance. The `require_fastbinary=False` option should only be used when system dependencies cannot be installed.
|
|
167
|
+
|
|
169
168
|
### Perform a Queries & Get Results
|
|
170
169
|
|
|
171
170
|
```python
|
|
@@ -512,35 +511,6 @@ pool.return_connection(conn)
|
|
|
512
511
|
pool.close_all()
|
|
513
512
|
```
|
|
514
513
|
|
|
515
|
-
##### Connection Pool with HTTPS (HAProxy)
|
|
516
|
-
|
|
517
|
-
```python
|
|
518
|
-
from e6data_python_connector import ConnectionPool
|
|
519
|
-
|
|
520
|
-
# Connection pool with HTTPS and custom CA certificate
|
|
521
|
-
pool = ConnectionPool(
|
|
522
|
-
min_size=2,
|
|
523
|
-
max_size=10,
|
|
524
|
-
host=host,
|
|
525
|
-
port=port,
|
|
526
|
-
username=username,
|
|
527
|
-
password=password,
|
|
528
|
-
database=database,
|
|
529
|
-
secure=True,
|
|
530
|
-
ssl_cert='/path/to/ca-cert.pem' # CA certificate for HTTPS
|
|
531
|
-
)
|
|
532
|
-
|
|
533
|
-
# Use the pool with context manager for automatic cleanup
|
|
534
|
-
with pool.get_connection_context() as conn:
|
|
535
|
-
cursor = conn.cursor()
|
|
536
|
-
cursor.execute("SELECT * FROM table")
|
|
537
|
-
results = cursor.fetchall()
|
|
538
|
-
print(f"Retrieved {len(results)} rows")
|
|
539
|
-
|
|
540
|
-
# Clean up pool when done
|
|
541
|
-
pool.close_all()
|
|
542
|
-
```
|
|
543
|
-
|
|
544
514
|
##### Using Context Manager (Recommended)
|
|
545
515
|
|
|
546
516
|
The context manager pattern ensures connections are automatically returned to the pool:
|
|
@@ -728,5 +698,6 @@ python your_script.py
|
|
|
728
698
|
| 456 Strategy Error | Automatic blue-green failover will handle this |
|
|
729
699
|
| Memory issues with large results | Use `fetchall_buffer()` instead of `fetchall()` |
|
|
730
700
|
| gRPC message size errors | Configure `grpc_options` with appropriate message size limits |
|
|
701
|
+
| fastbinary import error | Install system dependencies (`python3-devel`, `gcc-c++`) or set `require_fastbinary=False` |
|
|
731
702
|
|
|
732
703
|
See [TECH_DOC.md](TECH_DOC.md) for detailed technical documentation.
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import threading
|
|
3
3
|
import time
|
|
4
|
-
from typing import Optional, Union
|
|
5
4
|
import e6data_python_connector.cluster_server.cluster_pb2 as cluster_pb2
|
|
6
5
|
import e6data_python_connector.cluster_server.cluster_pb2_grpc as cluster_pb2_grpc
|
|
7
6
|
import grpc
|
|
@@ -141,7 +140,7 @@ class ClusterManager:
|
|
|
141
140
|
"""
|
|
142
141
|
|
|
143
142
|
def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 5,
|
|
144
|
-
cluster_uuid=None, grpc_options=None, debug=False
|
|
143
|
+
cluster_uuid=None, grpc_options=None, debug=False):
|
|
145
144
|
"""
|
|
146
145
|
Initializes a new instance of the ClusterManager class.
|
|
147
146
|
|
|
@@ -156,12 +155,7 @@ class ClusterManager:
|
|
|
156
155
|
defaults to 5 minutes.
|
|
157
156
|
cluster_uuid (str, optional): The unique identifier for the target cluster;
|
|
158
157
|
defaults to None.
|
|
159
|
-
grpc_options (dict, optional): Additional gRPC configuration options;
|
|
160
|
-
defaults to None.
|
|
161
158
|
debug (bool, optional): Enable debug logging; defaults to False.
|
|
162
|
-
ssl_cert (str or bytes, optional): Path to CA certificate file (PEM format)
|
|
163
|
-
or certificate content as bytes. Used when secure_channel=True.
|
|
164
|
-
If not provided and secure_channel=True, system default CA bundle will be used.
|
|
165
159
|
"""
|
|
166
160
|
|
|
167
161
|
self._host = host
|
|
@@ -170,67 +164,12 @@ class ClusterManager:
|
|
|
170
164
|
self._password = password
|
|
171
165
|
self._timeout = time.time() + timeout
|
|
172
166
|
self._secure_channel = secure_channel
|
|
173
|
-
self._ssl_cert = ssl_cert
|
|
174
167
|
self.cluster_uuid = cluster_uuid
|
|
175
168
|
self._grpc_options = grpc_options
|
|
176
169
|
if grpc_options is None:
|
|
177
170
|
self._grpc_options = dict()
|
|
178
171
|
self._debug = debug
|
|
179
172
|
|
|
180
|
-
def _get_ssl_credentials(self):
|
|
181
|
-
"""
|
|
182
|
-
Creates SSL/TLS credentials for secure gRPC connections.
|
|
183
|
-
|
|
184
|
-
This method handles loading CA certificates from various sources:
|
|
185
|
-
- File path (str): Reads the certificate from the specified file
|
|
186
|
-
- Bytes: Uses the certificate content directly
|
|
187
|
-
- None: Uses system default CA bundle
|
|
188
|
-
|
|
189
|
-
Returns:
|
|
190
|
-
grpc.ChannelCredentials: SSL credentials for secure channel creation
|
|
191
|
-
|
|
192
|
-
Raises:
|
|
193
|
-
FileNotFoundError: If ssl_cert is a file path and the file doesn't exist
|
|
194
|
-
ValueError: If ssl_cert format is invalid or file cannot be read
|
|
195
|
-
"""
|
|
196
|
-
root_certificates = None
|
|
197
|
-
|
|
198
|
-
if self._ssl_cert:
|
|
199
|
-
# Custom CA certificate provided
|
|
200
|
-
if isinstance(self._ssl_cert, str):
|
|
201
|
-
# It's a file path - read the certificate
|
|
202
|
-
try:
|
|
203
|
-
with open(self._ssl_cert, 'rb') as f:
|
|
204
|
-
root_certificates = f.read()
|
|
205
|
-
if self._debug:
|
|
206
|
-
logger.info(f"ClusterManager: Loaded CA certificate from file: {self._ssl_cert}")
|
|
207
|
-
except FileNotFoundError:
|
|
208
|
-
raise FileNotFoundError(
|
|
209
|
-
f"CA certificate file not found: {self._ssl_cert}. "
|
|
210
|
-
f"Please ensure the file exists and the path is correct."
|
|
211
|
-
)
|
|
212
|
-
except Exception as e:
|
|
213
|
-
raise ValueError(
|
|
214
|
-
f"Failed to read CA certificate from {self._ssl_cert}: {str(e)}"
|
|
215
|
-
)
|
|
216
|
-
elif isinstance(self._ssl_cert, bytes):
|
|
217
|
-
# Certificate content provided as bytes
|
|
218
|
-
root_certificates = self._ssl_cert
|
|
219
|
-
if self._debug:
|
|
220
|
-
logger.info("ClusterManager: Using CA certificate provided as bytes")
|
|
221
|
-
else:
|
|
222
|
-
raise ValueError(
|
|
223
|
-
f"ssl_cert must be either a file path (str) or certificate content (bytes), "
|
|
224
|
-
f"got {type(self._ssl_cert)}"
|
|
225
|
-
)
|
|
226
|
-
else:
|
|
227
|
-
# No custom CA certificate - use system default CA bundle
|
|
228
|
-
if self._debug:
|
|
229
|
-
logger.info("ClusterManager: Using system default CA bundle for SSL/TLS")
|
|
230
|
-
|
|
231
|
-
# Create and return SSL credentials with certificate verification enabled
|
|
232
|
-
return grpc.ssl_channel_credentials(root_certificates=root_certificates)
|
|
233
|
-
|
|
234
173
|
@property
|
|
235
174
|
def _get_connection(self):
|
|
236
175
|
"""
|
|
@@ -242,29 +181,16 @@ class ClusterManager:
|
|
|
242
181
|
"""
|
|
243
182
|
|
|
244
183
|
if self._secure_channel:
|
|
245
|
-
# Get SSL credentials (handles CA cert loading and validation)
|
|
246
|
-
credentials = self._get_ssl_credentials()
|
|
247
|
-
|
|
248
184
|
self._channel = grpc.secure_channel(
|
|
249
185
|
target='{}:{}'.format(self._host, self._port),
|
|
250
186
|
options=self._grpc_options,
|
|
251
|
-
credentials=
|
|
187
|
+
credentials=grpc.ssl_channel_credentials()
|
|
252
188
|
)
|
|
253
|
-
|
|
254
|
-
if self._debug:
|
|
255
|
-
logger.info(
|
|
256
|
-
f"ClusterManager: Created secure gRPC channel to {self._host}:{self._port} "
|
|
257
|
-
f"with SSL/TLS certificate verification enabled"
|
|
258
|
-
)
|
|
259
189
|
else:
|
|
260
190
|
self._channel = grpc.insecure_channel(
|
|
261
191
|
target='{}:{}'.format(self._host, self._port),
|
|
262
192
|
options=self._grpc_options
|
|
263
193
|
)
|
|
264
|
-
|
|
265
|
-
if self._debug:
|
|
266
|
-
logger.info(f"ClusterManager: Created insecure gRPC channel to {self._host}:{self._port}")
|
|
267
|
-
|
|
268
194
|
return cluster_pb2_grpc.ClusterServiceStub(self._channel)
|
|
269
195
|
|
|
270
196
|
def _try_cluster_request(self, request_type, payload=None):
|
|
@@ -12,20 +12,27 @@ from e6data_python_connector.e6x_vector.ttypes import Chunk, Vector, VectorType
|
|
|
12
12
|
from e6data_python_connector.constants import ZONE
|
|
13
13
|
from e6data_python_connector.date_time_utils import floor_div, floor_mod, timezone_from_offset
|
|
14
14
|
|
|
15
|
+
# Try to import fastbinary - it's optional but provides better performance
|
|
16
|
+
_fastbinary_available = False
|
|
15
17
|
try:
|
|
16
18
|
from thrift.protocol import fastbinary
|
|
19
|
+
_fastbinary_available = True
|
|
17
20
|
except ImportError:
|
|
18
|
-
|
|
19
|
-
"""
|
|
20
|
-
Failed to import fastbinary.
|
|
21
|
-
Did you install system dependencies?
|
|
22
|
-
Please verify https://github.com/e6x-labs/e6data-python-connector#dependencies
|
|
23
|
-
"""
|
|
24
|
-
)
|
|
21
|
+
pass # Will check require_fastbinary flag at connection creation time
|
|
25
22
|
|
|
26
23
|
_logger = logging.getLogger(__name__)
|
|
27
24
|
|
|
28
25
|
|
|
26
|
+
def is_fastbinary_available():
|
|
27
|
+
"""
|
|
28
|
+
Check if fastbinary module is available.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
bool: True if fastbinary is available, False otherwise
|
|
32
|
+
"""
|
|
33
|
+
return _fastbinary_available
|
|
34
|
+
|
|
35
|
+
|
|
29
36
|
def _binary_to_decimal128(binary_data, scale=None):
|
|
30
37
|
"""
|
|
31
38
|
Convert binary data to Decimal128.
|
|
@@ -504,6 +511,16 @@ def read_values_from_array(query_columns_description: list, dis: DataInputStream
|
|
|
504
511
|
|
|
505
512
|
|
|
506
513
|
def read_rows_from_chunk(query_columns_description: list, buffer):
|
|
514
|
+
"""
|
|
515
|
+
Read rows from a Thrift-encoded chunk buffer.
|
|
516
|
+
|
|
517
|
+
Args:
|
|
518
|
+
query_columns_description: List of column descriptions
|
|
519
|
+
buffer: Thrift-encoded binary buffer
|
|
520
|
+
|
|
521
|
+
Returns:
|
|
522
|
+
List of rows
|
|
523
|
+
"""
|
|
507
524
|
# Create a transport and protocol instance for deserialization
|
|
508
525
|
transport = TTransport.TMemoryBuffer(buffer)
|
|
509
526
|
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
|
|
@@ -16,7 +16,6 @@ import time
|
|
|
16
16
|
from decimal import Decimal
|
|
17
17
|
from io import BytesIO
|
|
18
18
|
from ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
|
|
19
|
-
from typing import Union, Optional
|
|
20
19
|
import threading
|
|
21
20
|
import multiprocessing
|
|
22
21
|
|
|
@@ -27,7 +26,7 @@ from e6data_python_connector.cluster_manager import ClusterManager
|
|
|
27
26
|
from e6data_python_connector.strategy import _get_grpc_header as _get_strategy_header
|
|
28
27
|
from e6data_python_connector.common import DBAPITypeObject, ParamEscaper, DBAPICursor
|
|
29
28
|
from e6data_python_connector.constants import *
|
|
30
|
-
from e6data_python_connector.datainputstream import get_query_columns_info, read_rows_from_chunk
|
|
29
|
+
from e6data_python_connector.datainputstream import get_query_columns_info, read_rows_from_chunk, is_fastbinary_available
|
|
31
30
|
from e6data_python_connector.server import e6x_engine_pb2_grpc, e6x_engine_pb2
|
|
32
31
|
from e6data_python_connector.typeId import *
|
|
33
32
|
|
|
@@ -336,7 +335,7 @@ class Connection(object):
|
|
|
336
335
|
scheme: str = 'e6data',
|
|
337
336
|
grpc_options: dict = None,
|
|
338
337
|
debug: bool = False,
|
|
339
|
-
|
|
338
|
+
require_fastbinary: bool = True,
|
|
340
339
|
):
|
|
341
340
|
"""
|
|
342
341
|
Parameters
|
|
@@ -370,14 +369,10 @@ class Connection(object):
|
|
|
370
369
|
- keepalive_time_ms: This parameter defines the time, in milliseconds, Default to 30 seconds
|
|
371
370
|
debug: bool, Optional
|
|
372
371
|
Flag to enable debug logging for blue-green deployment strategy changes
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
For self-signed certificates, provide the CA certificate via this parameter.
|
|
378
|
-
Examples:
|
|
379
|
-
- File path: ssl_cert='/path/to/ca-cert.pem'
|
|
380
|
-
- Bytes: ssl_cert=open('/path/to/ca-cert.pem', 'rb').read()
|
|
372
|
+
require_fastbinary: bool, Optional
|
|
373
|
+
Flag to require fastbinary module for Thrift deserialization. If True (default),
|
|
374
|
+
raises an exception if fastbinary is not available. If False, logs a warning
|
|
375
|
+
and continues with pure Python implementation (with reduced performance).
|
|
381
376
|
"""
|
|
382
377
|
if not username or not password:
|
|
383
378
|
raise ValueError("username or password cannot be empty.")
|
|
@@ -392,12 +387,34 @@ class Connection(object):
|
|
|
392
387
|
self._port = port
|
|
393
388
|
|
|
394
389
|
self._secure_channel = secure
|
|
395
|
-
self._ssl_cert = ssl_cert
|
|
396
390
|
|
|
397
391
|
self.catalog_name = catalog
|
|
398
392
|
|
|
399
393
|
self._auto_resume = auto_resume
|
|
400
394
|
|
|
395
|
+
# Store require_fastbinary flag
|
|
396
|
+
self._require_fastbinary = require_fastbinary
|
|
397
|
+
|
|
398
|
+
# Check fastbinary availability at connection creation time
|
|
399
|
+
if not is_fastbinary_available():
|
|
400
|
+
if require_fastbinary:
|
|
401
|
+
raise Exception(
|
|
402
|
+
"""
|
|
403
|
+
Failed to import fastbinary.
|
|
404
|
+
Did you install system dependencies?
|
|
405
|
+
Please verify https://github.com/e6x-labs/e6data-python-connector#dependencies
|
|
406
|
+
|
|
407
|
+
To continue without fastbinary (with reduced performance), set require_fastbinary=False
|
|
408
|
+
in the connection parameters.
|
|
409
|
+
"""
|
|
410
|
+
)
|
|
411
|
+
else:
|
|
412
|
+
logger.warning(
|
|
413
|
+
"fastbinary module is not available. Using pure Python implementation. "
|
|
414
|
+
"Performance may be degraded. To enable fastbinary, install system dependencies: "
|
|
415
|
+
"https://github.com/e6x-labs/e6data-python-connector#dependencies"
|
|
416
|
+
)
|
|
417
|
+
|
|
401
418
|
self._grpc_options = grpc_options
|
|
402
419
|
if self._grpc_options is None:
|
|
403
420
|
self._grpc_options = dict()
|
|
@@ -494,60 +511,6 @@ class Connection(object):
|
|
|
494
511
|
|
|
495
512
|
return self._cached_grpc_options
|
|
496
513
|
|
|
497
|
-
def _get_ssl_credentials(self):
|
|
498
|
-
"""
|
|
499
|
-
Creates SSL/TLS credentials for secure gRPC connections.
|
|
500
|
-
|
|
501
|
-
This method handles loading CA certificates from various sources:
|
|
502
|
-
- File path (str): Reads the certificate from the specified file
|
|
503
|
-
- Bytes: Uses the certificate content directly
|
|
504
|
-
- None: Uses system default CA bundle
|
|
505
|
-
|
|
506
|
-
Returns:
|
|
507
|
-
grpc.ChannelCredentials: SSL credentials for secure channel creation
|
|
508
|
-
|
|
509
|
-
Raises:
|
|
510
|
-
FileNotFoundError: If ssl_cert is a file path and the file doesn't exist
|
|
511
|
-
ValueError: If ssl_cert format is invalid or file cannot be read
|
|
512
|
-
"""
|
|
513
|
-
root_certificates = None
|
|
514
|
-
|
|
515
|
-
if self._ssl_cert:
|
|
516
|
-
# Custom CA certificate provided
|
|
517
|
-
if isinstance(self._ssl_cert, str):
|
|
518
|
-
# It's a file path - read the certificate
|
|
519
|
-
try:
|
|
520
|
-
with open(self._ssl_cert, 'rb') as f:
|
|
521
|
-
root_certificates = f.read()
|
|
522
|
-
if self._debug:
|
|
523
|
-
logger.info(f"Loaded CA certificate from file: {self._ssl_cert}")
|
|
524
|
-
except FileNotFoundError:
|
|
525
|
-
raise FileNotFoundError(
|
|
526
|
-
f"CA certificate file not found: {self._ssl_cert}. "
|
|
527
|
-
f"Please ensure the file exists and the path is correct."
|
|
528
|
-
)
|
|
529
|
-
except Exception as e:
|
|
530
|
-
raise ValueError(
|
|
531
|
-
f"Failed to read CA certificate from {self._ssl_cert}: {str(e)}"
|
|
532
|
-
)
|
|
533
|
-
elif isinstance(self._ssl_cert, bytes):
|
|
534
|
-
# Certificate content provided as bytes
|
|
535
|
-
root_certificates = self._ssl_cert
|
|
536
|
-
if self._debug:
|
|
537
|
-
logger.info("Using CA certificate provided as bytes")
|
|
538
|
-
else:
|
|
539
|
-
raise ValueError(
|
|
540
|
-
f"ssl_cert must be either a file path (str) or certificate content (bytes), "
|
|
541
|
-
f"got {type(self._ssl_cert)}"
|
|
542
|
-
)
|
|
543
|
-
else:
|
|
544
|
-
# No custom CA certificate - use system default CA bundle
|
|
545
|
-
if self._debug:
|
|
546
|
-
logger.info("Using system default CA bundle for SSL/TLS")
|
|
547
|
-
|
|
548
|
-
# Create and return SSL credentials with certificate verification enabled
|
|
549
|
-
return grpc.ssl_channel_credentials(root_certificates=root_certificates)
|
|
550
|
-
|
|
551
514
|
def _create_client(self):
|
|
552
515
|
"""
|
|
553
516
|
Creates a gRPC client for the connection.
|
|
@@ -562,19 +525,13 @@ class Connection(object):
|
|
|
562
525
|
|
|
563
526
|
Raises:
|
|
564
527
|
grpc.RpcError: If there is an error in creating the gRPC channel or client stub.
|
|
565
|
-
FileNotFoundError: If ssl_cert is a file path and the file doesn't exist.
|
|
566
|
-
ValueError: If ssl_cert format is invalid.
|
|
567
528
|
"""
|
|
568
529
|
|
|
569
530
|
if self._secure_channel:
|
|
570
|
-
# Get SSL credentials (handles CA cert loading and validation)
|
|
571
|
-
credentials = self._get_ssl_credentials()
|
|
572
|
-
|
|
573
|
-
# Create secure channel
|
|
574
531
|
self._channel = grpc.secure_channel(
|
|
575
532
|
target='{}:{}'.format(self._host, self._port),
|
|
576
533
|
options=self._get_grpc_options,
|
|
577
|
-
credentials=
|
|
534
|
+
credentials=grpc.ssl_channel_credentials()
|
|
578
535
|
)
|
|
579
536
|
else:
|
|
580
537
|
self._channel = grpc.insecure_channel(
|
|
@@ -756,8 +713,7 @@ class Connection(object):
|
|
|
756
713
|
secure_channel=self._secure_channel,
|
|
757
714
|
cluster_uuid=self.cluster_name,
|
|
758
715
|
timeout=self.grpc_auto_resume_timeout_seconds,
|
|
759
|
-
debug=self._debug
|
|
760
|
-
ssl_cert=self._ssl_cert
|
|
716
|
+
debug=self._debug
|
|
761
717
|
).resume()
|
|
762
718
|
return status # Return boolean status directly
|
|
763
719
|
else:
|
|
@@ -1536,7 +1492,10 @@ class Cursor(DBAPICursor):
|
|
|
1536
1492
|
if not buffer or len(buffer) == 0:
|
|
1537
1493
|
return None
|
|
1538
1494
|
# one batch retrieves the predefined set of rows
|
|
1539
|
-
return read_rows_from_chunk(
|
|
1495
|
+
return read_rows_from_chunk(
|
|
1496
|
+
self._query_columns_description,
|
|
1497
|
+
buffer
|
|
1498
|
+
)
|
|
1540
1499
|
|
|
1541
1500
|
def fetchall(self):
|
|
1542
1501
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.12rc31
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
6
|
Author: e6data, Inc.
|
|
@@ -39,7 +39,7 @@ Dynamic: summary
|
|
|
39
39
|
|
|
40
40
|
# e6data Python Connector
|
|
41
41
|
|
|
42
|
-

|
|
43
43
|
|
|
44
44
|
## Introduction
|
|
45
45
|
|
|
@@ -125,7 +125,7 @@ The `Connection` class supports the following parameters:
|
|
|
125
125
|
| `auto_resume` | bool | No | True | Automatically resume cluster if suspended |
|
|
126
126
|
| `grpc_options` | dict | No | None | Additional gRPC configuration options |
|
|
127
127
|
| `debug` | bool | No | False | Enable debug logging for troubleshooting |
|
|
128
|
-
| `
|
|
128
|
+
| `require_fastbinary` | bool | No | True | Require fastbinary module for Thrift deserialization. Set to False to use pure Python implementation if system dependencies cannot be installed |
|
|
129
129
|
|
|
130
130
|
#### Secure Connection Example
|
|
131
131
|
|
|
@@ -143,14 +143,11 @@ conn = Connection(
|
|
|
143
143
|
)
|
|
144
144
|
```
|
|
145
145
|
|
|
146
|
-
####
|
|
146
|
+
#### Cluster-Specific Connection
|
|
147
147
|
|
|
148
|
-
When
|
|
148
|
+
When working with multiple clusters, specify the cluster name:
|
|
149
149
|
|
|
150
150
|
```python
|
|
151
|
-
from e6data_python_connector import Connection
|
|
152
|
-
|
|
153
|
-
# Option 1: Using CA certificate file path
|
|
154
151
|
conn = Connection(
|
|
155
152
|
host=host,
|
|
156
153
|
port=port,
|
|
@@ -158,40 +155,30 @@ conn = Connection(
|
|
|
158
155
|
password=password,
|
|
159
156
|
database=database,
|
|
160
157
|
cluster_name='analytics-cluster-01', # Specify cluster name
|
|
161
|
-
secure=True
|
|
162
|
-
ssl_cert='/path/to/ca-cert.pem' # Path to CA certificate
|
|
158
|
+
secure=True
|
|
163
159
|
)
|
|
160
|
+
```
|
|
164
161
|
|
|
165
|
-
|
|
166
|
-
with open('/path/to/ca-cert.pem', 'rb') as f:
|
|
167
|
-
ca_cert = f.read()
|
|
162
|
+
#### Handling Missing System Dependencies
|
|
168
163
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
password=password,
|
|
174
|
-
database=database,
|
|
175
|
-
secure=True,
|
|
176
|
-
ssl_cert=ca_cert # Certificate as bytes
|
|
177
|
-
)
|
|
164
|
+
The e6data connector uses the `fastbinary` module (from Apache Thrift) for optimal performance when deserializing data. This module requires system-level dependencies (`python3-devel` and `gcc-c++`) to be installed.
|
|
165
|
+
|
|
166
|
+
**Default Behavior (Recommended):**
|
|
167
|
+
By default, the connector requires `fastbinary` to be available. If it's not found, the connection will fail immediately with a clear error message:
|
|
178
168
|
|
|
179
|
-
|
|
169
|
+
```python
|
|
180
170
|
conn = Connection(
|
|
181
171
|
host=host,
|
|
182
172
|
port=port,
|
|
183
173
|
username=username,
|
|
184
174
|
password=password,
|
|
185
|
-
database=database
|
|
186
|
-
secure=True # Uses system CA bundle
|
|
175
|
+
database=database
|
|
187
176
|
)
|
|
177
|
+
# Raises exception if fastbinary is not available
|
|
188
178
|
```
|
|
189
179
|
|
|
190
|
-
**
|
|
191
|
-
|
|
192
|
-
#### Cluster-Specific Connection
|
|
193
|
-
|
|
194
|
-
When working with multiple clusters, specify the cluster name:
|
|
180
|
+
**Fallback to Pure Python:**
|
|
181
|
+
If you cannot install system dependencies (e.g., in restricted environments, serverless platforms, or containers without build tools), you can disable the `fastbinary` requirement. The connector will fall back to a pure Python implementation with a performance penalty:
|
|
195
182
|
|
|
196
183
|
```python
|
|
197
184
|
conn = Connection(
|
|
@@ -200,11 +187,23 @@ conn = Connection(
|
|
|
200
187
|
username=username,
|
|
201
188
|
password=password,
|
|
202
189
|
database=database,
|
|
203
|
-
|
|
204
|
-
secure=True
|
|
190
|
+
require_fastbinary=False # Allow operation without fastbinary
|
|
205
191
|
)
|
|
192
|
+
# Logs warning but continues with pure Python implementation
|
|
206
193
|
```
|
|
207
194
|
|
|
195
|
+
**When to use `require_fastbinary=False`:**
|
|
196
|
+
- Running in AWS Lambda or other serverless environments
|
|
197
|
+
- Docker containers built without compilation tools
|
|
198
|
+
- Restricted environments where system packages cannot be installed
|
|
199
|
+
- Development/testing environments where performance is not critical
|
|
200
|
+
|
|
201
|
+
**Performance Impact:**
|
|
202
|
+
- With `fastbinary`: Optimal performance for data deserialization
|
|
203
|
+
- Without `fastbinary` (pure Python): ~2-3x slower deserialization, but otherwise fully functional
|
|
204
|
+
|
|
205
|
+
**Note:** It's strongly recommended to install system dependencies when possible for best performance. The `require_fastbinary=False` option should only be used when system dependencies cannot be installed.
|
|
206
|
+
|
|
208
207
|
### Perform a Queries & Get Results
|
|
209
208
|
|
|
210
209
|
```python
|
|
@@ -551,35 +550,6 @@ pool.return_connection(conn)
|
|
|
551
550
|
pool.close_all()
|
|
552
551
|
```
|
|
553
552
|
|
|
554
|
-
##### Connection Pool with HTTPS (HAProxy)
|
|
555
|
-
|
|
556
|
-
```python
|
|
557
|
-
from e6data_python_connector import ConnectionPool
|
|
558
|
-
|
|
559
|
-
# Connection pool with HTTPS and custom CA certificate
|
|
560
|
-
pool = ConnectionPool(
|
|
561
|
-
min_size=2,
|
|
562
|
-
max_size=10,
|
|
563
|
-
host=host,
|
|
564
|
-
port=port,
|
|
565
|
-
username=username,
|
|
566
|
-
password=password,
|
|
567
|
-
database=database,
|
|
568
|
-
secure=True,
|
|
569
|
-
ssl_cert='/path/to/ca-cert.pem' # CA certificate for HTTPS
|
|
570
|
-
)
|
|
571
|
-
|
|
572
|
-
# Use the pool with context manager for automatic cleanup
|
|
573
|
-
with pool.get_connection_context() as conn:
|
|
574
|
-
cursor = conn.cursor()
|
|
575
|
-
cursor.execute("SELECT * FROM table")
|
|
576
|
-
results = cursor.fetchall()
|
|
577
|
-
print(f"Retrieved {len(results)} rows")
|
|
578
|
-
|
|
579
|
-
# Clean up pool when done
|
|
580
|
-
pool.close_all()
|
|
581
|
-
```
|
|
582
|
-
|
|
583
553
|
##### Using Context Manager (Recommended)
|
|
584
554
|
|
|
585
555
|
The context manager pattern ensures connections are automatically returned to the pool:
|
|
@@ -767,5 +737,6 @@ python your_script.py
|
|
|
767
737
|
| 456 Strategy Error | Automatic blue-green failover will handle this |
|
|
768
738
|
| Memory issues with large results | Use `fetchall_buffer()` instead of `fetchall()` |
|
|
769
739
|
| gRPC message size errors | Configure `grpc_options` with appropriate message size limits |
|
|
740
|
+
| fastbinary import error | Install system dependencies (`python3-devel`, `gcc-c++`) or set `require_fastbinary=False` |
|
|
770
741
|
|
|
771
742
|
See [TECH_DOC.md](TECH_DOC.md) for detailed technical documentation.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/gen-py/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_38_nines.py
RENAMED
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_all_cases.py
RENAMED
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_binary.py
RENAMED
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/analyze_fields.py
RENAMED
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/cleanup_test_files.py
RENAMED
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/debug_38_nines.py
RENAMED
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/debug_binary.py
RENAMED
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/final_test.py
RENAMED
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/move_tests.py
RENAMED
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/quick_test.py
RENAMED
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_38_nines.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_comprehensive.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_known_case.py
RENAMED
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_mock_server.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/test_strategy.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/tests_grpc.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.3.12rc12 → e6data_python_connector-2.3.12rc31}/test/verify_decimal_fix.py
RENAMED
|
File without changes
|