e6data-python-connector 2.3.11rc2__py3-none-any.whl → 2.3.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- e6data_python_connector/cluster_manager.py +7 -22
- e6data_python_connector/common.py +44 -0
- e6data_python_connector/constants.py +24 -0
- e6data_python_connector/datainputstream.py +55 -26
- e6data_python_connector/e6data_grpc.py +100 -21
- e6data_python_connector/e6x_vector/ttypes.py +26 -2
- {e6data_python_connector-2.3.11rc2.dist-info → e6data_python_connector-2.3.12.dist-info}/METADATA +171 -2
- {e6data_python_connector-2.3.11rc2.dist-info → e6data_python_connector-2.3.12.dist-info}/RECORD +16 -12
- test/benchmark_fastbinary_performance.py +303 -0
- test/test_code_optimizations.py +322 -0
- test/test_optimizations_integration.py +443 -0
- test/test_ssl_cert.py +548 -0
- {e6data_python_connector-2.3.11rc2.dist-info → e6data_python_connector-2.3.12.dist-info}/LICENSE +0 -0
- {e6data_python_connector-2.3.11rc2.dist-info → e6data_python_connector-2.3.12.dist-info}/WHEEL +0 -0
- {e6data_python_connector-2.3.11rc2.dist-info → e6data_python_connector-2.3.12.dist-info}/entry_points.txt +0 -0
- {e6data_python_connector-2.3.11rc2.dist-info → e6data_python_connector-2.3.12.dist-info}/top_level.txt +0 -0
|
@@ -9,27 +9,9 @@ import multiprocessing
|
|
|
9
9
|
|
|
10
10
|
logger = logging.getLogger(__name__)
|
|
11
11
|
|
|
12
|
+
from e6data_python_connector.common import get_ssl_credentials
|
|
12
13
|
from e6data_python_connector.strategy import _get_active_strategy, _set_active_strategy, _set_pending_strategy, \
|
|
13
|
-
_get_grpc_header
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def _get_grpc_header(engine_ip=None, cluster=None, strategy=None):
|
|
17
|
-
"""
|
|
18
|
-
Generate gRPC metadata headers for the request.
|
|
19
|
-
|
|
20
|
-
This function creates a list of metadata headers to be used in gRPC requests.
|
|
21
|
-
It includes optional headers for the engine IP, cluster UUID, and deployment strategy.
|
|
22
|
-
|
|
23
|
-
Args:
|
|
24
|
-
engine_ip (str, optional): The IP address of the engine. Defaults to None.
|
|
25
|
-
cluster (str, optional): The UUID of the cluster. Defaults to None.
|
|
26
|
-
strategy (str, optional): The deployment strategy (blue/green). Defaults to None.
|
|
27
|
-
|
|
28
|
-
Returns:
|
|
29
|
-
list: A list of tuples representing the gRPC metadata headers.
|
|
30
|
-
"""
|
|
31
|
-
# Use the strategy module's implementation
|
|
32
|
-
return _get_strategy_header(engine_ip=engine_ip, cluster=cluster, strategy=strategy)
|
|
14
|
+
_get_grpc_header
|
|
33
15
|
|
|
34
16
|
|
|
35
17
|
class _StatusLock:
|
|
@@ -140,7 +122,7 @@ class ClusterManager:
|
|
|
140
122
|
"""
|
|
141
123
|
|
|
142
124
|
def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 5,
|
|
143
|
-
cluster_uuid=None, grpc_options=None, debug=False):
|
|
125
|
+
cluster_uuid=None, grpc_options=None, debug=False, ssl_cert=None):
|
|
144
126
|
"""
|
|
145
127
|
Initializes a new instance of the ClusterManager class.
|
|
146
128
|
|
|
@@ -156,6 +138,8 @@ class ClusterManager:
|
|
|
156
138
|
cluster_uuid (str, optional): The unique identifier for the target cluster;
|
|
157
139
|
defaults to None.
|
|
158
140
|
debug (bool, optional): Enable debug logging; defaults to False.
|
|
141
|
+
ssl_cert (str or bytes, optional): Path to CA certificate file (PEM format) or
|
|
142
|
+
certificate content as bytes for secure connections; defaults to None.
|
|
159
143
|
"""
|
|
160
144
|
|
|
161
145
|
self._host = host
|
|
@@ -169,6 +153,7 @@ class ClusterManager:
|
|
|
169
153
|
if grpc_options is None:
|
|
170
154
|
self._grpc_options = dict()
|
|
171
155
|
self._debug = debug
|
|
156
|
+
self._ssl_cert = ssl_cert
|
|
172
157
|
|
|
173
158
|
@property
|
|
174
159
|
def _get_connection(self):
|
|
@@ -184,7 +169,7 @@ class ClusterManager:
|
|
|
184
169
|
self._channel = grpc.secure_channel(
|
|
185
170
|
target='{}:{}'.format(self._host, self._port),
|
|
186
171
|
options=self._grpc_options,
|
|
187
|
-
credentials=
|
|
172
|
+
credentials=get_ssl_credentials(self._ssl_cert)
|
|
188
173
|
)
|
|
189
174
|
else:
|
|
190
175
|
self._channel = grpc.insecure_channel(
|
|
@@ -15,6 +15,7 @@ from builtins import int
|
|
|
15
15
|
from builtins import object
|
|
16
16
|
from builtins import str
|
|
17
17
|
|
|
18
|
+
import grpc
|
|
18
19
|
from future.utils import with_metaclass
|
|
19
20
|
from past.builtins import basestring
|
|
20
21
|
|
|
@@ -267,3 +268,46 @@ class UniversalSet(object):
|
|
|
267
268
|
|
|
268
269
|
def __contains__(self, item):
|
|
269
270
|
return True
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def get_ssl_credentials(ssl_cert):
|
|
274
|
+
"""
|
|
275
|
+
Get SSL credentials for secure gRPC channel.
|
|
276
|
+
|
|
277
|
+
Handles three scenarios:
|
|
278
|
+
1. ssl_cert is a string (file path): Read the PEM certificate from the file
|
|
279
|
+
2. ssl_cert is bytes: Use the certificate content directly
|
|
280
|
+
3. ssl_cert is None: Use system default CA bundle
|
|
281
|
+
|
|
282
|
+
Parameters:
|
|
283
|
+
ssl_cert (str or bytes or None): SSL certificate as file path, bytes, or None
|
|
284
|
+
|
|
285
|
+
Returns:
|
|
286
|
+
grpc.ChannelCredentials: SSL credentials for gRPC channel
|
|
287
|
+
|
|
288
|
+
Raises:
|
|
289
|
+
FileNotFoundError: If ssl_cert is a file path but the file doesn't exist
|
|
290
|
+
IOError: If ssl_cert file cannot be read
|
|
291
|
+
"""
|
|
292
|
+
if ssl_cert is None:
|
|
293
|
+
# Use system default CA bundle
|
|
294
|
+
return grpc.ssl_channel_credentials()
|
|
295
|
+
elif isinstance(ssl_cert, str):
|
|
296
|
+
# ssl_cert is a file path - read the certificate from file
|
|
297
|
+
try:
|
|
298
|
+
with open(ssl_cert, 'rb') as cert_file:
|
|
299
|
+
root_ca_cert = cert_file.read()
|
|
300
|
+
return grpc.ssl_channel_credentials(root_certificates=root_ca_cert)
|
|
301
|
+
except FileNotFoundError:
|
|
302
|
+
_logger.error("SSL certificate file not found: {}".format(ssl_cert))
|
|
303
|
+
raise
|
|
304
|
+
except IOError as e:
|
|
305
|
+
_logger.error("Failed to read SSL certificate file {}: {}".format(ssl_cert, e))
|
|
306
|
+
raise
|
|
307
|
+
elif isinstance(ssl_cert, bytes):
|
|
308
|
+
# ssl_cert is certificate content as bytes
|
|
309
|
+
return grpc.ssl_channel_credentials(root_certificates=ssl_cert)
|
|
310
|
+
else:
|
|
311
|
+
# Invalid type - fall back to system default CA bundle with warning
|
|
312
|
+
_logger.warning("Invalid ssl_cert type: {}. Using system default CA bundle.".format(type(ssl_cert)))
|
|
313
|
+
return grpc.ssl_channel_credentials()
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import pytz
|
|
2
2
|
|
|
3
|
+
# Type definitions
|
|
3
4
|
PRIMITIVE_TYPES = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 18, 19, 20, 21}
|
|
4
5
|
COMPLEX_TYPES = {10, 11, 12, 13, 14}
|
|
5
6
|
COLLECTION_TYPES = {10, 11}
|
|
@@ -30,3 +31,26 @@ CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength"
|
|
|
30
31
|
PRECISION = "precision"
|
|
31
32
|
SCALE = "scale"
|
|
32
33
|
ZONE = pytz.timezone('UTC')
|
|
34
|
+
|
|
35
|
+
# Retry and timeout constants
|
|
36
|
+
MAX_RETRY_ATTEMPTS = 5
|
|
37
|
+
RETRY_SLEEP_SECONDS = 0.2
|
|
38
|
+
STRATEGY_CACHE_TIMEOUT_SECONDS = 300 # 5 minutes
|
|
39
|
+
DEFAULT_GRPC_PREPARE_TIMEOUT_SECONDS = 600 # 10 minutes
|
|
40
|
+
DEFAULT_AUTO_RESUME_TIMEOUT_SECONDS = 300 # 5 minutes
|
|
41
|
+
CLUSTER_STATUS_CHECK_SLEEP_SECONDS = 5
|
|
42
|
+
LOCK_TIMEOUT_MS = 500
|
|
43
|
+
|
|
44
|
+
# Connection pool constants
|
|
45
|
+
POOL_GET_TIMEOUT_SECONDS = 0.1
|
|
46
|
+
POOL_RETRY_SLEEP_SECONDS = 0.1
|
|
47
|
+
|
|
48
|
+
# Blue-green deployment strategy constants
|
|
49
|
+
STRATEGY_BLUE = 'blue'
|
|
50
|
+
STRATEGY_GREEN = 'green'
|
|
51
|
+
VALID_STRATEGIES = {STRATEGY_BLUE, STRATEGY_GREEN}
|
|
52
|
+
|
|
53
|
+
# gRPC error codes and messages
|
|
54
|
+
GRPC_ERROR_STRATEGY_MISMATCH = 'status: 456'
|
|
55
|
+
GRPC_ERROR_SERVICE_UNAVAILABLE = 'status: 503'
|
|
56
|
+
GRPC_ERROR_ACCESS_DENIED = 'Access denied'
|
|
@@ -12,30 +12,39 @@ from e6data_python_connector.e6x_vector.ttypes import Chunk, Vector, VectorType
|
|
|
12
12
|
from e6data_python_connector.constants import ZONE
|
|
13
13
|
from e6data_python_connector.date_time_utils import floor_div, floor_mod, timezone_from_offset
|
|
14
14
|
|
|
15
|
+
# Try to import fastbinary - it's optional but provides better performance
|
|
16
|
+
_fastbinary_available = False
|
|
15
17
|
try:
|
|
16
18
|
from thrift.protocol import fastbinary
|
|
19
|
+
_fastbinary_available = True
|
|
17
20
|
except ImportError:
|
|
18
|
-
|
|
19
|
-
"""
|
|
20
|
-
Failed to import fastbinary.
|
|
21
|
-
Did you install system dependencies?
|
|
22
|
-
Please verify https://github.com/e6x-labs/e6data-python-connector#dependencies
|
|
23
|
-
"""
|
|
24
|
-
)
|
|
21
|
+
pass # Will check require_fastbinary flag at connection creation time
|
|
25
22
|
|
|
26
23
|
_logger = logging.getLogger(__name__)
|
|
27
24
|
|
|
28
25
|
|
|
29
|
-
def
|
|
26
|
+
def is_fastbinary_available():
|
|
27
|
+
"""
|
|
28
|
+
Check if fastbinary module is available.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
bool: True if fastbinary is available, False otherwise
|
|
32
|
+
"""
|
|
33
|
+
return _fastbinary_available
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _binary_to_decimal128(binary_data, scale=None):
|
|
30
37
|
"""
|
|
31
38
|
Convert binary data to Decimal128.
|
|
32
|
-
|
|
39
|
+
|
|
33
40
|
The binary data represents a 128-bit decimal number in IEEE 754-2008 Decimal128 format.
|
|
34
41
|
Based on the Java implementation from e6data's JDBC driver.
|
|
35
|
-
|
|
42
|
+
|
|
36
43
|
Args:
|
|
37
44
|
binary_data (bytes): Binary representation of Decimal128
|
|
38
|
-
|
|
45
|
+
scale (int, optional): Scale parameter for decimal precision. If None,
|
|
46
|
+
attempts to decode from IEEE format or defaults to 0.
|
|
47
|
+
|
|
39
48
|
Returns:
|
|
40
49
|
Decimal: Python Decimal object
|
|
41
50
|
"""
|
|
@@ -59,7 +68,7 @@ def _binary_to_decimal128(binary_data):
|
|
|
59
68
|
|
|
60
69
|
# Handle IEEE 754-2008 Decimal128 binary format
|
|
61
70
|
if len(binary_data) == 16: # Decimal128 should be exactly 16 bytes
|
|
62
|
-
return _decode_decimal128_binary_java_style(binary_data)
|
|
71
|
+
return _decode_decimal128_binary_java_style(binary_data, scale)
|
|
63
72
|
else:
|
|
64
73
|
_logger.warning(f"Invalid Decimal128 binary length: {len(binary_data)} bytes, expected 16")
|
|
65
74
|
return Decimal('0')
|
|
@@ -73,16 +82,17 @@ def _binary_to_decimal128(binary_data):
|
|
|
73
82
|
return Decimal('0')
|
|
74
83
|
|
|
75
84
|
|
|
76
|
-
def _decode_decimal128_binary_java_style(binary_data):
|
|
85
|
+
def _decode_decimal128_binary_java_style(binary_data, scale=None):
|
|
77
86
|
"""
|
|
78
87
|
Decode IEEE 754-2008 Decimal128 binary format following Java implementation.
|
|
79
|
-
|
|
88
|
+
|
|
80
89
|
Based on the Java implementation from e6data's JDBC driver getFieldDataFromChunk method.
|
|
81
90
|
This method follows the same logic as the Java BigDecimal creation from ByteBuffer.
|
|
82
|
-
|
|
91
|
+
|
|
83
92
|
Args:
|
|
84
93
|
binary_data (bytes): 16-byte binary representation
|
|
85
|
-
|
|
94
|
+
scale (int, optional): Scale parameter for decimal precision
|
|
95
|
+
|
|
86
96
|
Returns:
|
|
87
97
|
Decimal: Python Decimal object
|
|
88
98
|
"""
|
|
@@ -103,17 +113,21 @@ def _decode_decimal128_binary_java_style(binary_data):
|
|
|
103
113
|
if big_int_value == 0:
|
|
104
114
|
return Decimal('0')
|
|
105
115
|
|
|
106
|
-
# The Java code creates BigDecimal from BigInteger with scale
|
|
107
|
-
#
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
116
|
+
# The Java code creates BigDecimal from BigInteger with the provided scale
|
|
117
|
+
# If scale is provided, use it to create the properly scaled decimal
|
|
118
|
+
if scale is not None:
|
|
119
|
+
# Create decimal with the specified scale (like Java's BigDecimal constructor)
|
|
120
|
+
# This treats big_int_value as the unscaled value
|
|
121
|
+
decimal_value = Decimal(big_int_value) / (Decimal(10) ** scale)
|
|
122
|
+
else:
|
|
123
|
+
# Fallback: try to create decimal directly from the integer value
|
|
124
|
+
decimal_value = Decimal(big_int_value)
|
|
112
125
|
|
|
113
126
|
# Check if this produces a reasonable decimal value
|
|
114
127
|
# Decimal128 should represent normal decimal numbers
|
|
115
|
-
if
|
|
116
|
-
|
|
128
|
+
# Only check range if scale was not provided (backward compatibility)
|
|
129
|
+
if scale is None and (abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal(
|
|
130
|
+
'9.999999999999999999999999999999999E+6144')):
|
|
117
131
|
# Value is outside normal Decimal128 range, try alternative interpretation
|
|
118
132
|
return _decode_decimal128_alternative(binary_data)
|
|
119
133
|
|
|
@@ -497,6 +511,16 @@ def read_values_from_array(query_columns_description: list, dis: DataInputStream
|
|
|
497
511
|
|
|
498
512
|
|
|
499
513
|
def read_rows_from_chunk(query_columns_description: list, buffer):
|
|
514
|
+
"""
|
|
515
|
+
Read rows from a Thrift-encoded chunk buffer.
|
|
516
|
+
|
|
517
|
+
Args:
|
|
518
|
+
query_columns_description: List of column descriptions
|
|
519
|
+
buffer: Thrift-encoded binary buffer
|
|
520
|
+
|
|
521
|
+
Returns:
|
|
522
|
+
List of rows
|
|
523
|
+
"""
|
|
500
524
|
# Create a transport and protocol instance for deserialization
|
|
501
525
|
transport = TTransport.TMemoryBuffer(buffer)
|
|
502
526
|
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
|
|
@@ -624,10 +648,12 @@ def get_column_from_chunk(vector: Vector) -> list:
|
|
|
624
648
|
if vector.isConstantVector:
|
|
625
649
|
# For constant vectors, get the binary data and convert it once
|
|
626
650
|
binary_data = vector.data.numericDecimal128ConstantData.data
|
|
651
|
+
# Get scale with backward compatibility for older engines
|
|
652
|
+
scale = getattr(vector.data.numericDecimal128ConstantData, 'scale', None)
|
|
627
653
|
|
|
628
654
|
# Convert binary data to BigDecimal equivalent
|
|
629
655
|
if binary_data:
|
|
630
|
-
decimal_value = _binary_to_decimal128(binary_data)
|
|
656
|
+
decimal_value = _binary_to_decimal128(binary_data, scale)
|
|
631
657
|
else:
|
|
632
658
|
decimal_value = Decimal('0')
|
|
633
659
|
|
|
@@ -639,13 +665,16 @@ def get_column_from_chunk(vector: Vector) -> list:
|
|
|
639
665
|
value_array.append(decimal_value)
|
|
640
666
|
else:
|
|
641
667
|
# For non-constant vectors, process each row individually
|
|
668
|
+
# Get scale from decimal128Data (with backward compatibility)
|
|
669
|
+
scale = getattr(vector.data.decimal128Data, 'scale', None)
|
|
670
|
+
|
|
642
671
|
for row in range(vector.size):
|
|
643
672
|
if get_null(vector, row):
|
|
644
673
|
value_array.append(None)
|
|
645
674
|
continue
|
|
646
675
|
# Get binary data for this row
|
|
647
676
|
binary_data = vector.data.decimal128Data.data[row]
|
|
648
|
-
decimal_value = _binary_to_decimal128(binary_data)
|
|
677
|
+
decimal_value = _binary_to_decimal128(binary_data, scale)
|
|
649
678
|
value_array.append(decimal_value)
|
|
650
679
|
else:
|
|
651
680
|
value_array.append(None)
|
|
@@ -8,24 +8,28 @@ from __future__ import unicode_literals
|
|
|
8
8
|
|
|
9
9
|
import datetime
|
|
10
10
|
import logging
|
|
11
|
+
import os
|
|
11
12
|
import re
|
|
12
13
|
import sys
|
|
14
|
+
import threading
|
|
13
15
|
import time
|
|
14
16
|
from decimal import Decimal
|
|
15
17
|
from io import BytesIO
|
|
16
18
|
from ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
|
|
17
|
-
import threading
|
|
18
|
-
import multiprocessing
|
|
19
19
|
|
|
20
20
|
import grpc
|
|
21
21
|
from grpc._channel import _InactiveRpcError
|
|
22
22
|
|
|
23
23
|
from e6data_python_connector.cluster_manager import ClusterManager
|
|
24
|
-
from e6data_python_connector.
|
|
25
|
-
from e6data_python_connector.
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
from e6data_python_connector.common import DBAPITypeObject, ParamEscaper, DBAPICursor, get_ssl_credentials
|
|
25
|
+
from e6data_python_connector.constants import (
|
|
26
|
+
MAX_RETRY_ATTEMPTS, RETRY_SLEEP_SECONDS, GRPC_ERROR_STRATEGY_MISMATCH, GRPC_ERROR_ACCESS_DENIED,
|
|
27
|
+
PRIMITIVE_TYPES
|
|
28
|
+
)
|
|
29
|
+
from e6data_python_connector.datainputstream import get_query_columns_info, read_rows_from_chunk, \
|
|
30
|
+
is_fastbinary_available
|
|
28
31
|
from e6data_python_connector.server import e6x_engine_pb2_grpc, e6x_engine_pb2
|
|
32
|
+
from e6data_python_connector.strategy import _get_grpc_header
|
|
29
33
|
from e6data_python_connector.typeId import *
|
|
30
34
|
|
|
31
35
|
apilevel = '2.0'
|
|
@@ -69,7 +73,7 @@ TYPES_CONVERTER = {
|
|
|
69
73
|
|
|
70
74
|
def re_auth(func):
|
|
71
75
|
def wrapper(self, *args, **kwargs):
|
|
72
|
-
max_retry =
|
|
76
|
+
max_retry = MAX_RETRY_ATTEMPTS
|
|
73
77
|
current_retry = 0
|
|
74
78
|
while current_retry < max_retry:
|
|
75
79
|
try:
|
|
@@ -78,10 +82,10 @@ def re_auth(func):
|
|
|
78
82
|
current_retry += 1
|
|
79
83
|
if current_retry == max_retry:
|
|
80
84
|
raise e
|
|
81
|
-
if e.code() == grpc.StatusCode.INTERNAL and
|
|
82
|
-
time.sleep(
|
|
85
|
+
if e.code() == grpc.StatusCode.INTERNAL and GRPC_ERROR_ACCESS_DENIED in e.details():
|
|
86
|
+
time.sleep(RETRY_SLEEP_SECONDS)
|
|
83
87
|
self.connection.get_re_authenticate_session_id()
|
|
84
|
-
elif
|
|
88
|
+
elif GRPC_ERROR_STRATEGY_MISMATCH in e.details():
|
|
85
89
|
# Strategy changed, clear cache and retry
|
|
86
90
|
_clear_strategy_cache()
|
|
87
91
|
# Force re-authentication which will detect new strategy
|
|
@@ -302,12 +306,6 @@ def _get_strategy_debug_info():
|
|
|
302
306
|
}
|
|
303
307
|
|
|
304
308
|
|
|
305
|
-
def _get_grpc_header(engine_ip=None, cluster=None, strategy=None):
|
|
306
|
-
"""Generate gRPC metadata headers for the request."""
|
|
307
|
-
# Use the strategy module's implementation
|
|
308
|
-
return _get_strategy_header(engine_ip=engine_ip, cluster=cluster, strategy=strategy)
|
|
309
|
-
|
|
310
|
-
|
|
311
309
|
def connect(*args, **kwargs):
|
|
312
310
|
"""Constructor for creating a connection to the database. See class :py:class:`Connection` for
|
|
313
311
|
arguments.
|
|
@@ -329,10 +327,12 @@ class Connection(object):
|
|
|
329
327
|
database: str = None,
|
|
330
328
|
cluster_name: str = None,
|
|
331
329
|
secure: bool = False,
|
|
330
|
+
ssl_cert = None,
|
|
332
331
|
auto_resume: bool = True,
|
|
333
332
|
scheme: str = 'e6data',
|
|
334
333
|
grpc_options: dict = None,
|
|
335
334
|
debug: bool = False,
|
|
335
|
+
require_fastbinary: bool = True,
|
|
336
336
|
):
|
|
337
337
|
"""
|
|
338
338
|
Parameters
|
|
@@ -353,6 +353,9 @@ class Connection(object):
|
|
|
353
353
|
Cluster's name
|
|
354
354
|
secure: bool, Optional
|
|
355
355
|
Flag to use a secure channel for data transfer
|
|
356
|
+
ssl_cert: str or bytes, Optional
|
|
357
|
+
Path to CA certificate file (PEM format) or certificate content as bytes for
|
|
358
|
+
secure connections. If None, system default CA bundle is used.
|
|
356
359
|
auto_resume: bool, Optional
|
|
357
360
|
Flag to enable auto resume of the cluster before the query execution
|
|
358
361
|
scheme: string, Optional
|
|
@@ -366,6 +369,10 @@ class Connection(object):
|
|
|
366
369
|
- keepalive_time_ms: This parameter defines the time, in milliseconds, Default to 30 seconds
|
|
367
370
|
debug: bool, Optional
|
|
368
371
|
Flag to enable debug logging for blue-green deployment strategy changes
|
|
372
|
+
require_fastbinary: bool, Optional
|
|
373
|
+
Flag to require fastbinary module for Thrift deserialization. If True (default),
|
|
374
|
+
raises an exception if fastbinary is not available. If False, logs a warning
|
|
375
|
+
and continues with pure Python implementation (with reduced performance).
|
|
369
376
|
"""
|
|
370
377
|
if not username or not password:
|
|
371
378
|
raise ValueError("username or password cannot be empty.")
|
|
@@ -380,11 +387,35 @@ class Connection(object):
|
|
|
380
387
|
self._port = port
|
|
381
388
|
|
|
382
389
|
self._secure_channel = secure
|
|
390
|
+
self._ssl_cert = ssl_cert
|
|
383
391
|
|
|
384
392
|
self.catalog_name = catalog
|
|
385
393
|
|
|
386
394
|
self._auto_resume = auto_resume
|
|
387
395
|
|
|
396
|
+
# Store require_fastbinary flag
|
|
397
|
+
self._require_fastbinary = require_fastbinary
|
|
398
|
+
|
|
399
|
+
# Check fastbinary availability at connection creation time
|
|
400
|
+
if not is_fastbinary_available():
|
|
401
|
+
if require_fastbinary:
|
|
402
|
+
raise Exception(
|
|
403
|
+
"""
|
|
404
|
+
Failed to import fastbinary.
|
|
405
|
+
Did you install system dependencies?
|
|
406
|
+
Please verify https://github.com/e6x-labs/e6data-python-connector#dependencies
|
|
407
|
+
|
|
408
|
+
To continue without fastbinary (with reduced performance), set require_fastbinary=False
|
|
409
|
+
in the connection parameters.
|
|
410
|
+
"""
|
|
411
|
+
)
|
|
412
|
+
else:
|
|
413
|
+
logger.warning(
|
|
414
|
+
"fastbinary module is not available. Using pure Python implementation. "
|
|
415
|
+
"Performance may be degraded. To enable fastbinary, install system dependencies: "
|
|
416
|
+
"https://github.com/e6x-labs/e6data-python-connector#dependencies"
|
|
417
|
+
)
|
|
418
|
+
|
|
388
419
|
self._grpc_options = grpc_options
|
|
389
420
|
if self._grpc_options is None:
|
|
390
421
|
self._grpc_options = dict()
|
|
@@ -400,8 +431,51 @@ class Connection(object):
|
|
|
400
431
|
self._debug = debug
|
|
401
432
|
if self._debug:
|
|
402
433
|
_debug_connections.add(id(self))
|
|
434
|
+
|
|
435
|
+
# Enable comprehensive debugging if debug flag is set
|
|
436
|
+
if self._debug:
|
|
437
|
+
# Configure root logger for DEBUG level
|
|
438
|
+
logging.basicConfig(
|
|
439
|
+
level=logging.DEBUG,
|
|
440
|
+
format='[%(name)s] %(asctime)s - %(levelname)s - %(message)s',
|
|
441
|
+
datefmt='%Y-%m-%d %H:%M:%S',
|
|
442
|
+
force=True # Force reconfiguration even if logging is already configured
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
# Note: gRPC C++ core tracing (GRPC_VERBOSITY and GRPC_TRACE) must be set
|
|
446
|
+
# BEFORE the gRPC module is imported to take effect. Setting them at runtime
|
|
447
|
+
# will not enable HTTP/2 frame logs or low-level tracing.
|
|
448
|
+
#
|
|
449
|
+
# To enable full gRPC network tracing, set these environment variables
|
|
450
|
+
# before starting your Python script:
|
|
451
|
+
# export GRPC_VERBOSITY=DEBUG
|
|
452
|
+
# export GRPC_TRACE=client_channel,http2
|
|
453
|
+
#
|
|
454
|
+
# The following runtime settings only affect Python-level logging:
|
|
455
|
+
|
|
456
|
+
# Enable gRPC Python logging (this works at runtime)
|
|
457
|
+
os.environ['GRPC_PYTHON_LOG_LEVEL'] = 'DEBUG'
|
|
458
|
+
os.environ['GRPC_PYTHON_LOG_STDERR'] = '1'
|
|
459
|
+
|
|
460
|
+
# Ensure gRPC logger is at DEBUG level
|
|
461
|
+
grpc_logger = logging.getLogger('grpc')
|
|
462
|
+
grpc_logger.setLevel(logging.DEBUG)
|
|
463
|
+
|
|
464
|
+
# Enable gRPC transport logger
|
|
465
|
+
grpc_transport_logger = logging.getLogger('grpc._channel')
|
|
466
|
+
grpc_transport_logger.setLevel(logging.DEBUG)
|
|
467
|
+
|
|
468
|
+
# Enable gRPC server logger
|
|
469
|
+
grpc_server_logger = logging.getLogger('grpc._server')
|
|
470
|
+
grpc_server_logger.setLevel(logging.DEBUG)
|
|
471
|
+
|
|
472
|
+
# Set e6data connector logger to DEBUG
|
|
473
|
+
e6data_logger = logging.getLogger('e6data_python_connector')
|
|
474
|
+
e6data_logger.setLevel(logging.DEBUG)
|
|
475
|
+
|
|
403
476
|
_strategy_debug_log(f"Debug mode enabled for connection {id(self)}")
|
|
404
|
-
|
|
477
|
+
_strategy_debug_log(f"GRPC_TRACE={os.environ.get('GRPC_TRACE')}")
|
|
478
|
+
|
|
405
479
|
self._create_client()
|
|
406
480
|
|
|
407
481
|
@property
|
|
@@ -453,11 +527,12 @@ class Connection(object):
|
|
|
453
527
|
Raises:
|
|
454
528
|
grpc.RpcError: If there is an error in creating the gRPC channel or client stub.
|
|
455
529
|
"""
|
|
530
|
+
|
|
456
531
|
if self._secure_channel:
|
|
457
532
|
self._channel = grpc.secure_channel(
|
|
458
533
|
target='{}:{}'.format(self._host, self._port),
|
|
459
534
|
options=self._get_grpc_options,
|
|
460
|
-
credentials=
|
|
535
|
+
credentials=get_ssl_credentials(self._ssl_cert)
|
|
461
536
|
)
|
|
462
537
|
else:
|
|
463
538
|
self._channel = grpc.insecure_channel(
|
|
@@ -577,7 +652,7 @@ class Connection(object):
|
|
|
577
652
|
_strategy_debug_log(f"No cached strategy, will try strategies in order: {strategies}")
|
|
578
653
|
last_error = None
|
|
579
654
|
for strategy in strategies:
|
|
580
|
-
_strategy_debug_log(f"
|
|
655
|
+
_strategy_debug_log(f"Attempting authentication with strategy: {strategy}.")
|
|
581
656
|
try:
|
|
582
657
|
authenticate_response = self._client.authenticate(
|
|
583
658
|
authenticate_request,
|
|
@@ -639,7 +714,8 @@ class Connection(object):
|
|
|
639
714
|
secure_channel=self._secure_channel,
|
|
640
715
|
cluster_uuid=self.cluster_name,
|
|
641
716
|
timeout=self.grpc_auto_resume_timeout_seconds,
|
|
642
|
-
debug=self._debug
|
|
717
|
+
debug=self._debug,
|
|
718
|
+
ssl_cert=self._ssl_cert
|
|
643
719
|
).resume()
|
|
644
720
|
return status # Return boolean status directly
|
|
645
721
|
else:
|
|
@@ -1418,7 +1494,10 @@ class Cursor(DBAPICursor):
|
|
|
1418
1494
|
if not buffer or len(buffer) == 0:
|
|
1419
1495
|
return None
|
|
1420
1496
|
# one batch retrieves the predefined set of rows
|
|
1421
|
-
return read_rows_from_chunk(
|
|
1497
|
+
return read_rows_from_chunk(
|
|
1498
|
+
self._query_columns_description,
|
|
1499
|
+
buffer
|
|
1500
|
+
)
|
|
1422
1501
|
|
|
1423
1502
|
def fetchall(self):
|
|
1424
1503
|
"""
|
|
@@ -881,13 +881,15 @@ class Decimal128Data(object):
|
|
|
881
881
|
"""
|
|
882
882
|
Attributes:
|
|
883
883
|
- data
|
|
884
|
+
- scale
|
|
884
885
|
|
|
885
886
|
"""
|
|
886
887
|
thrift_spec = None
|
|
887
888
|
|
|
888
889
|
|
|
889
|
-
def __init__(self, data = None,):
|
|
890
|
+
def __init__(self, data = None, scale = None,):
|
|
890
891
|
self.data = data
|
|
892
|
+
self.scale = scale
|
|
891
893
|
|
|
892
894
|
def read(self, iprot):
|
|
893
895
|
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
@@ -908,6 +910,11 @@ class Decimal128Data(object):
|
|
|
908
910
|
iprot.readListEnd()
|
|
909
911
|
else:
|
|
910
912
|
iprot.skip(ftype)
|
|
913
|
+
elif fid == 2:
|
|
914
|
+
if ftype == TType.I32:
|
|
915
|
+
self.scale = iprot.readI32()
|
|
916
|
+
else:
|
|
917
|
+
iprot.skip(ftype)
|
|
911
918
|
else:
|
|
912
919
|
iprot.skip(ftype)
|
|
913
920
|
iprot.readFieldEnd()
|
|
@@ -926,6 +933,10 @@ class Decimal128Data(object):
|
|
|
926
933
|
oprot.writeBinary(iter55)
|
|
927
934
|
oprot.writeListEnd()
|
|
928
935
|
oprot.writeFieldEnd()
|
|
936
|
+
if self.scale is not None:
|
|
937
|
+
oprot.writeFieldBegin('scale', TType.I32, 2)
|
|
938
|
+
oprot.writeI32(self.scale)
|
|
939
|
+
oprot.writeFieldEnd()
|
|
929
940
|
oprot.writeFieldStop()
|
|
930
941
|
oprot.writeStructEnd()
|
|
931
942
|
|
|
@@ -1310,13 +1321,15 @@ class NumericDecimal128ConstantData(object):
|
|
|
1310
1321
|
"""
|
|
1311
1322
|
Attributes:
|
|
1312
1323
|
- data
|
|
1324
|
+
- scale
|
|
1313
1325
|
|
|
1314
1326
|
"""
|
|
1315
1327
|
thrift_spec = None
|
|
1316
1328
|
|
|
1317
1329
|
|
|
1318
|
-
def __init__(self, data = None,):
|
|
1330
|
+
def __init__(self, data = None, scale = None,):
|
|
1319
1331
|
self.data = data
|
|
1332
|
+
self.scale = scale
|
|
1320
1333
|
|
|
1321
1334
|
def read(self, iprot):
|
|
1322
1335
|
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
@@ -1332,6 +1345,11 @@ class NumericDecimal128ConstantData(object):
|
|
|
1332
1345
|
self.data = iprot.readBinary()
|
|
1333
1346
|
else:
|
|
1334
1347
|
iprot.skip(ftype)
|
|
1348
|
+
elif fid == 2:
|
|
1349
|
+
if ftype == TType.I32:
|
|
1350
|
+
self.scale = iprot.readI32()
|
|
1351
|
+
else:
|
|
1352
|
+
iprot.skip(ftype)
|
|
1335
1353
|
else:
|
|
1336
1354
|
iprot.skip(ftype)
|
|
1337
1355
|
iprot.readFieldEnd()
|
|
@@ -1347,6 +1365,10 @@ class NumericDecimal128ConstantData(object):
|
|
|
1347
1365
|
oprot.writeFieldBegin('data', TType.STRING, 1)
|
|
1348
1366
|
oprot.writeBinary(self.data)
|
|
1349
1367
|
oprot.writeFieldEnd()
|
|
1368
|
+
if self.scale is not None:
|
|
1369
|
+
oprot.writeFieldBegin('scale', TType.I32, 2)
|
|
1370
|
+
oprot.writeI32(self.scale)
|
|
1371
|
+
oprot.writeFieldEnd()
|
|
1350
1372
|
oprot.writeFieldStop()
|
|
1351
1373
|
oprot.writeStructEnd()
|
|
1352
1374
|
|
|
@@ -1783,6 +1805,7 @@ all_structs.append(Decimal128Data)
|
|
|
1783
1805
|
Decimal128Data.thrift_spec = (
|
|
1784
1806
|
None, # 0
|
|
1785
1807
|
(1, TType.LIST, 'data', (TType.STRING, 'BINARY', False), None, ), # 1
|
|
1808
|
+
(2, TType.I32, 'scale', None, None, ), # 2
|
|
1786
1809
|
)
|
|
1787
1810
|
all_structs.append(VarcharData)
|
|
1788
1811
|
VarcharData.thrift_spec = (
|
|
@@ -1818,6 +1841,7 @@ all_structs.append(NumericDecimal128ConstantData)
|
|
|
1818
1841
|
NumericDecimal128ConstantData.thrift_spec = (
|
|
1819
1842
|
None, # 0
|
|
1820
1843
|
(1, TType.STRING, 'data', 'BINARY', None, ), # 1
|
|
1844
|
+
(2, TType.I32, 'scale', None, None, ), # 2
|
|
1821
1845
|
)
|
|
1822
1846
|
all_structs.append(TemporalIntervalConstantData)
|
|
1823
1847
|
TemporalIntervalConstantData.thrift_spec = (
|