e6data-python-connector 2.3.12rc32__tar.gz → 2.3.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {e6data_python_connector-2.3.12rc32/e6data_python_connector.egg-info → e6data_python_connector-2.3.13}/PKG-INFO +6 -2
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/README.md +5 -1
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/cluster_manager.py +3 -60
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/common.py +44 -0
- e6data_python_connector-2.3.13/e6data_python_connector/constants.py +56 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/datainputstream.py +59 -36
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/date_time_utils.py +171 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/e6data_grpc.py +14 -57
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13/e6data_python_connector.egg-info}/PKG-INFO +6 -2
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector.egg-info/SOURCES.txt +1 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/setup.py +1 -1
- e6data_python_connector-2.3.13/test/test_code_optimizations.py +322 -0
- e6data_python_connector-2.3.12rc32/e6data_python_connector/constants.py +0 -32
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/LICENSE +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/MANIFEST.in +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/cluster_server/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/cluster_server/cluster_pb2.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/cluster_server/cluster_pb2_grpc.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/connection_pool.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/dialect.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/e6x_vector/constants.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/e6x_vector/ttypes.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/exceptions.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/server/QueryEngineService.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/server/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/server/constants.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/server/e6x_engine_pb2.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/server/e6x_engine_pb2.pyi +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/server/e6x_engine_pb2_grpc.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/server/ttypes.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/strategy.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector/typeId.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector.egg-info/dependency_links.txt +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector.egg-info/entry_points.txt +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector.egg-info/requires.txt +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/e6data_python_connector.egg-info/top_level.txt +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/gen-py/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/gen-py/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/gen-py/e6x_vector/constants.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/gen-py/e6x_vector/ttypes.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/pyproject.toml +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/setup.cfg +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/__init__.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/analyze_38_nines.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/analyze_all_cases.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/analyze_binary.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/analyze_correct_value.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/analyze_fields.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/check_decimal_errors.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/cleanup_test_files.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/debug_38_nines.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/debug_binary.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/final_test.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/move_tests.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/quick_test.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_38_nines.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_all_decimal128_cases.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_cluster_manager_efficiency.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_cluster_manager_none_strategy.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_cluster_manager_strategy.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_comprehensive.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_connection_pool.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_connection_pool_e2e.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_current_implementation.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_decimal128_binary_parsing.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_decimal128_parsing.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_fix.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_improved_parsing.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_known_case.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_manual_analysis.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_mock_server.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_multiprocessing_fix.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_new_implementation.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_pool_concurrency_simple.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_pool_threading_multiprocessing.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_simple_connection_pool.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_specific_binary.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_strategy.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_strategy_logic.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_strategy_persistence_fix.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_strategy_sharing_fix.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/test_user_binary_value.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/tests.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/tests_grpc.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/validate_decimal128.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/validate_implementation.py +0 -0
- {e6data_python_connector-2.3.12rc32 → e6data_python_connector-2.3.13}/test/verify_decimal_fix.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.13
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
6
|
Author: e6data, Inc.
|
|
@@ -39,7 +39,7 @@ Dynamic: summary
|
|
|
39
39
|
|
|
40
40
|
# e6data Python Connector
|
|
41
41
|
|
|
42
|
-

|
|
43
43
|
|
|
44
44
|
## Introduction
|
|
45
45
|
|
|
@@ -54,6 +54,10 @@ yum install python3-devel gcc-c++ -y
|
|
|
54
54
|
# Ubuntu/Debian dependencies
|
|
55
55
|
apt install python3-dev g++ -y
|
|
56
56
|
|
|
57
|
+
# Windows dependencies
|
|
58
|
+
# Install Visual C++ Build Tools from:
|
|
59
|
+
# https://visualstudio.microsoft.com/visual-cpp-build-tools/
|
|
60
|
+
# Select the "Desktop Development with C++" option during installation.
|
|
57
61
|
|
|
58
62
|
# Pip dependencies
|
|
59
63
|
pip install wheel
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# e6data Python Connector
|
|
2
2
|
|
|
3
|
-

|
|
4
4
|
|
|
5
5
|
## Introduction
|
|
6
6
|
|
|
@@ -15,6 +15,10 @@ yum install python3-devel gcc-c++ -y
|
|
|
15
15
|
# Ubuntu/Debian dependencies
|
|
16
16
|
apt install python3-dev g++ -y
|
|
17
17
|
|
|
18
|
+
# Windows dependencies
|
|
19
|
+
# Install Visual C++ Build Tools from:
|
|
20
|
+
# https://visualstudio.microsoft.com/visual-cpp-build-tools/
|
|
21
|
+
# Select the "Desktop Development with C++" option during installation.
|
|
18
22
|
|
|
19
23
|
# Pip dependencies
|
|
20
24
|
pip install wheel
|
|
@@ -9,27 +9,9 @@ import multiprocessing
|
|
|
9
9
|
|
|
10
10
|
logger = logging.getLogger(__name__)
|
|
11
11
|
|
|
12
|
+
from e6data_python_connector.common import get_ssl_credentials
|
|
12
13
|
from e6data_python_connector.strategy import _get_active_strategy, _set_active_strategy, _set_pending_strategy, \
|
|
13
|
-
_get_grpc_header
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def _get_grpc_header(engine_ip=None, cluster=None, strategy=None):
|
|
17
|
-
"""
|
|
18
|
-
Generate gRPC metadata headers for the request.
|
|
19
|
-
|
|
20
|
-
This function creates a list of metadata headers to be used in gRPC requests.
|
|
21
|
-
It includes optional headers for the engine IP, cluster UUID, and deployment strategy.
|
|
22
|
-
|
|
23
|
-
Args:
|
|
24
|
-
engine_ip (str, optional): The IP address of the engine. Defaults to None.
|
|
25
|
-
cluster (str, optional): The UUID of the cluster. Defaults to None.
|
|
26
|
-
strategy (str, optional): The deployment strategy (blue/green). Defaults to None.
|
|
27
|
-
|
|
28
|
-
Returns:
|
|
29
|
-
list: A list of tuples representing the gRPC metadata headers.
|
|
30
|
-
"""
|
|
31
|
-
# Use the strategy module's implementation
|
|
32
|
-
return _get_strategy_header(engine_ip=engine_ip, cluster=cluster, strategy=strategy)
|
|
14
|
+
_get_grpc_header
|
|
33
15
|
|
|
34
16
|
|
|
35
17
|
class _StatusLock:
|
|
@@ -173,45 +155,6 @@ class ClusterManager:
|
|
|
173
155
|
self._debug = debug
|
|
174
156
|
self._ssl_cert = ssl_cert
|
|
175
157
|
|
|
176
|
-
def _get_ssl_credentials(self):
|
|
177
|
-
"""
|
|
178
|
-
Get SSL credentials for secure gRPC channel.
|
|
179
|
-
|
|
180
|
-
Handles three scenarios:
|
|
181
|
-
1. ssl_cert is a string (file path): Read the PEM certificate from the file
|
|
182
|
-
2. ssl_cert is bytes: Use the certificate content directly
|
|
183
|
-
3. ssl_cert is None: Use system default CA bundle
|
|
184
|
-
|
|
185
|
-
Returns:
|
|
186
|
-
grpc.ChannelCredentials: SSL credentials for secure channel
|
|
187
|
-
|
|
188
|
-
Raises:
|
|
189
|
-
FileNotFoundError: If ssl_cert is a file path but the file doesn't exist
|
|
190
|
-
IOError: If ssl_cert file cannot be read
|
|
191
|
-
"""
|
|
192
|
-
if self._ssl_cert is None:
|
|
193
|
-
# Use system default CA bundle
|
|
194
|
-
return grpc.ssl_channel_credentials()
|
|
195
|
-
elif isinstance(self._ssl_cert, str):
|
|
196
|
-
# ssl_cert is a file path - read the certificate from file
|
|
197
|
-
try:
|
|
198
|
-
with open(self._ssl_cert, 'rb') as cert_file:
|
|
199
|
-
root_ca_cert = cert_file.read()
|
|
200
|
-
return grpc.ssl_channel_credentials(root_certificates=root_ca_cert)
|
|
201
|
-
except FileNotFoundError:
|
|
202
|
-
logger.error(f"SSL certificate file not found: {self._ssl_cert}")
|
|
203
|
-
raise
|
|
204
|
-
except IOError as e:
|
|
205
|
-
logger.error(f"Failed to read SSL certificate file {self._ssl_cert}: {e}")
|
|
206
|
-
raise
|
|
207
|
-
elif isinstance(self._ssl_cert, bytes):
|
|
208
|
-
# ssl_cert is certificate content as bytes
|
|
209
|
-
return grpc.ssl_channel_credentials(root_certificates=self._ssl_cert)
|
|
210
|
-
else:
|
|
211
|
-
# Invalid type - log warning and use system default
|
|
212
|
-
logger.warning(f"Invalid ssl_cert type: {type(self._ssl_cert)}. Using system default CA bundle.")
|
|
213
|
-
return grpc.ssl_channel_credentials()
|
|
214
|
-
|
|
215
158
|
@property
|
|
216
159
|
def _get_connection(self):
|
|
217
160
|
"""
|
|
@@ -226,7 +169,7 @@ class ClusterManager:
|
|
|
226
169
|
self._channel = grpc.secure_channel(
|
|
227
170
|
target='{}:{}'.format(self._host, self._port),
|
|
228
171
|
options=self._grpc_options,
|
|
229
|
-
credentials=self.
|
|
172
|
+
credentials=get_ssl_credentials(self._ssl_cert)
|
|
230
173
|
)
|
|
231
174
|
else:
|
|
232
175
|
self._channel = grpc.insecure_channel(
|
|
@@ -15,6 +15,7 @@ from builtins import int
|
|
|
15
15
|
from builtins import object
|
|
16
16
|
from builtins import str
|
|
17
17
|
|
|
18
|
+
import grpc
|
|
18
19
|
from future.utils import with_metaclass
|
|
19
20
|
from past.builtins import basestring
|
|
20
21
|
|
|
@@ -267,3 +268,46 @@ class UniversalSet(object):
|
|
|
267
268
|
|
|
268
269
|
def __contains__(self, item):
|
|
269
270
|
return True
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def get_ssl_credentials(ssl_cert):
|
|
274
|
+
"""
|
|
275
|
+
Get SSL credentials for secure gRPC channel.
|
|
276
|
+
|
|
277
|
+
Handles three scenarios:
|
|
278
|
+
1. ssl_cert is a string (file path): Read the PEM certificate from the file
|
|
279
|
+
2. ssl_cert is bytes: Use the certificate content directly
|
|
280
|
+
3. ssl_cert is None: Use system default CA bundle
|
|
281
|
+
|
|
282
|
+
Parameters:
|
|
283
|
+
ssl_cert (str or bytes or None): SSL certificate as file path, bytes, or None
|
|
284
|
+
|
|
285
|
+
Returns:
|
|
286
|
+
grpc.ChannelCredentials: SSL credentials for gRPC channel
|
|
287
|
+
|
|
288
|
+
Raises:
|
|
289
|
+
FileNotFoundError: If ssl_cert is a file path but the file doesn't exist
|
|
290
|
+
IOError: If ssl_cert file cannot be read
|
|
291
|
+
"""
|
|
292
|
+
if ssl_cert is None:
|
|
293
|
+
# Use system default CA bundle
|
|
294
|
+
return grpc.ssl_channel_credentials()
|
|
295
|
+
elif isinstance(ssl_cert, str):
|
|
296
|
+
# ssl_cert is a file path - read the certificate from file
|
|
297
|
+
try:
|
|
298
|
+
with open(ssl_cert, 'rb') as cert_file:
|
|
299
|
+
root_ca_cert = cert_file.read()
|
|
300
|
+
return grpc.ssl_channel_credentials(root_certificates=root_ca_cert)
|
|
301
|
+
except FileNotFoundError:
|
|
302
|
+
_logger.error("SSL certificate file not found: {}".format(ssl_cert))
|
|
303
|
+
raise
|
|
304
|
+
except IOError as e:
|
|
305
|
+
_logger.error("Failed to read SSL certificate file {}: {}".format(ssl_cert, e))
|
|
306
|
+
raise
|
|
307
|
+
elif isinstance(ssl_cert, bytes):
|
|
308
|
+
# ssl_cert is certificate content as bytes
|
|
309
|
+
return grpc.ssl_channel_credentials(root_certificates=ssl_cert)
|
|
310
|
+
else:
|
|
311
|
+
# Invalid type - fall back to system default CA bundle with warning
|
|
312
|
+
_logger.warning("Invalid ssl_cert type: {}. Using system default CA bundle.".format(type(ssl_cert)))
|
|
313
|
+
return grpc.ssl_channel_credentials()
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import pytz
|
|
2
|
+
|
|
3
|
+
# Type definitions
|
|
4
|
+
PRIMITIVE_TYPES = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 18, 19, 20, 21}
|
|
5
|
+
COMPLEX_TYPES = {10, 11, 12, 13, 14}
|
|
6
|
+
COLLECTION_TYPES = {10, 11}
|
|
7
|
+
TYPE_NAMES = {
|
|
8
|
+
0: "BOOLEAN",
|
|
9
|
+
1: "TINYINT",
|
|
10
|
+
2: "SMALLINT",
|
|
11
|
+
3: "INT",
|
|
12
|
+
4: "BIGINT",
|
|
13
|
+
5: "FLOAT",
|
|
14
|
+
6: "DOUBLE",
|
|
15
|
+
7: "STRING",
|
|
16
|
+
8: "TIMESTAMP",
|
|
17
|
+
9: "BINARY",
|
|
18
|
+
10: "ARRAY",
|
|
19
|
+
11: "MAP",
|
|
20
|
+
12: "STRUCT",
|
|
21
|
+
13: "UNIONTYPE",
|
|
22
|
+
15: "DECIMAL",
|
|
23
|
+
16: "NULL",
|
|
24
|
+
17: "DATE",
|
|
25
|
+
18: "VARCHAR",
|
|
26
|
+
19: "CHAR",
|
|
27
|
+
20: "INTERVAL_YEAR_MONTH",
|
|
28
|
+
21: "INTERVAL_DAY_TIME",
|
|
29
|
+
}
|
|
30
|
+
CHARACTER_MAXIMUM_LENGTH = "characterMaximumLength"
|
|
31
|
+
PRECISION = "precision"
|
|
32
|
+
SCALE = "scale"
|
|
33
|
+
ZONE = pytz.timezone('UTC')
|
|
34
|
+
|
|
35
|
+
# Retry and timeout constants
|
|
36
|
+
MAX_RETRY_ATTEMPTS = 5
|
|
37
|
+
RETRY_SLEEP_SECONDS = 0.2
|
|
38
|
+
STRATEGY_CACHE_TIMEOUT_SECONDS = 300 # 5 minutes
|
|
39
|
+
DEFAULT_GRPC_PREPARE_TIMEOUT_SECONDS = 600 # 10 minutes
|
|
40
|
+
DEFAULT_AUTO_RESUME_TIMEOUT_SECONDS = 300 # 5 minutes
|
|
41
|
+
CLUSTER_STATUS_CHECK_SLEEP_SECONDS = 5
|
|
42
|
+
LOCK_TIMEOUT_MS = 500
|
|
43
|
+
|
|
44
|
+
# Connection pool constants
|
|
45
|
+
POOL_GET_TIMEOUT_SECONDS = 0.1
|
|
46
|
+
POOL_RETRY_SLEEP_SECONDS = 0.1
|
|
47
|
+
|
|
48
|
+
# Blue-green deployment strategy constants
|
|
49
|
+
STRATEGY_BLUE = 'blue'
|
|
50
|
+
STRATEGY_GREEN = 'green'
|
|
51
|
+
VALID_STRATEGIES = {STRATEGY_BLUE, STRATEGY_GREEN}
|
|
52
|
+
|
|
53
|
+
# gRPC error codes and messages
|
|
54
|
+
GRPC_ERROR_STRATEGY_MISMATCH = 'status: 456'
|
|
55
|
+
GRPC_ERROR_SERVICE_UNAVAILABLE = 'status: 503'
|
|
56
|
+
GRPC_ERROR_ACCESS_DENIED = 'Access denied'
|
|
@@ -10,7 +10,13 @@ from thrift.transport import TTransport
|
|
|
10
10
|
|
|
11
11
|
from e6data_python_connector.e6x_vector.ttypes import Chunk, Vector, VectorType
|
|
12
12
|
from e6data_python_connector.constants import ZONE
|
|
13
|
-
from e6data_python_connector.date_time_utils import
|
|
13
|
+
from e6data_python_connector.date_time_utils import (
|
|
14
|
+
floor_div,
|
|
15
|
+
floor_mod,
|
|
16
|
+
timezone_from_offset,
|
|
17
|
+
format_iso_date_from_epoch_micros,
|
|
18
|
+
format_iso_datetime_from_epoch_micros,
|
|
19
|
+
)
|
|
14
20
|
|
|
15
21
|
# Try to import fastbinary - it's optional but provides better performance
|
|
16
22
|
_fastbinary_available = False
|
|
@@ -463,16 +469,11 @@ def read_values_from_array(query_columns_description: list, dis: DataInputStream
|
|
|
463
469
|
if dtype == "LONG":
|
|
464
470
|
value_array.append(dis.read_long())
|
|
465
471
|
elif dtype == "DATE":
|
|
466
|
-
|
|
467
|
-
date = datetime.fromtimestamp(epoch_seconds, ZONE)
|
|
468
|
-
value_array.append(date.strftime("%Y-%m-%d"))
|
|
472
|
+
value_array.append(format_iso_date_from_epoch_micros(dis.read_long()))
|
|
469
473
|
elif dtype == "DATETIME":
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
date_time = datetime.fromtimestamp(epoch_seconds, ZONE)
|
|
474
|
-
date_time = date_time + timedelta(microseconds=micros_of_the_day)
|
|
475
|
-
value_array.append(date_time.strftime("%Y-%m-%d %H:%M:%S"))
|
|
474
|
+
value_array.append(format_iso_datetime_from_epoch_micros(
|
|
475
|
+
dis.read_long(), tz=ZONE, separator=' ',
|
|
476
|
+
include_millis=False, include_offset=False))
|
|
476
477
|
elif dtype == "STRING" or dtype == "ARRAY" or dtype == "MAP" or dtype == "STRUCT":
|
|
477
478
|
value_array.append(dis.read_utf().decode())
|
|
478
479
|
elif dtype == "INT":
|
|
@@ -560,27 +561,38 @@ def get_column_from_chunk(vector: Vector) -> list:
|
|
|
560
561
|
value_array.append(vector.data.int64Data.data[
|
|
561
562
|
row] if not vector.isConstantVector else vector.data.numericConstantData.data)
|
|
562
563
|
elif d_type == VectorType.DATE:
|
|
564
|
+
# Use the JDBC-parity formatter so years > 9999 emit "+YYYYY-MM-DD"
|
|
565
|
+
# instead of raising. Per-row try/except keeps a single bad value
|
|
566
|
+
# from truncating the column (which previously cascaded to an
|
|
567
|
+
# IndexError in read_rows_from_chunk).
|
|
563
568
|
for row in range(vector.size):
|
|
564
569
|
if get_null(vector, row):
|
|
565
570
|
value_array.append(None)
|
|
566
571
|
continue
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
+
try:
|
|
573
|
+
raw = (vector.data.dateData.data[row]
|
|
574
|
+
if not vector.isConstantVector
|
|
575
|
+
else vector.data.dateConstantData.data)
|
|
576
|
+
value_array.append(format_iso_date_from_epoch_micros(raw))
|
|
577
|
+
except Exception as e:
|
|
578
|
+
_logger.error("Failed to parse DATE row=%s: %s", row, e)
|
|
579
|
+
value_array.append('Failed to parse.')
|
|
572
580
|
elif d_type == VectorType.DATETIME:
|
|
581
|
+
# Default formatter knobs reproduce the prior datetime.isoformat
|
|
582
|
+
# output ("YYYY-MM-DDTHH:MM:SS.sss+HH:MM"). See DATE branch above
|
|
583
|
+
# for the rationale on per-row try/except.
|
|
573
584
|
for row in range(vector.size):
|
|
574
585
|
if get_null(vector, row):
|
|
575
586
|
value_array.append(None)
|
|
576
587
|
continue
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
588
|
+
try:
|
|
589
|
+
epoch_micros = (vector.data.timeData.data[row]
|
|
590
|
+
if not vector.isConstantVector
|
|
591
|
+
else vector.data.timeConstantData.data)
|
|
592
|
+
value_array.append(format_iso_datetime_from_epoch_micros(epoch_micros, tz=zone))
|
|
593
|
+
except Exception as e:
|
|
594
|
+
_logger.error("Failed to parse DATETIME row=%s: %s", row, e)
|
|
595
|
+
value_array.append('Failed to parse.')
|
|
584
596
|
elif d_type == VectorType.STRING or d_type == VectorType.ARRAY or d_type == VectorType.MAP or d_type == VectorType.STRUCT:
|
|
585
597
|
for row in range(vector.size):
|
|
586
598
|
if get_null(vector, row):
|
|
@@ -627,22 +639,27 @@ def get_column_from_chunk(vector: Vector) -> list:
|
|
|
627
639
|
for row in range(vector.size):
|
|
628
640
|
value_array.append(None)
|
|
629
641
|
elif d_type == VectorType.TIMESTAMP_TZ:
|
|
642
|
+
# row_zone is scoped to the row (not the outer `zone`) so a
|
|
643
|
+
# per-row zone resolution doesn't leak into later iterations.
|
|
630
644
|
for row in range(vector.size):
|
|
631
645
|
if get_null(vector, row):
|
|
632
646
|
value_array.append(None)
|
|
633
647
|
continue
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
648
|
+
try:
|
|
649
|
+
epoch_micros = (vector.data.timeData.data[row]
|
|
650
|
+
if not vector.isConstantVector
|
|
651
|
+
else vector.data.timeConstantData.data)
|
|
652
|
+
row_zone = zone
|
|
653
|
+
if ((vector.isConstantVector and vector.data.timeConstantData.zoneData is not None) or
|
|
654
|
+
(not vector.isConstantVector and vector.data.timeData.zoneData is not None)):
|
|
655
|
+
zone_id = (vector.data.timeData.zoneData[row]
|
|
656
|
+
if not vector.isConstantVector
|
|
657
|
+
else vector.data.timeConstantData.zoneData)
|
|
658
|
+
row_zone = timezone_from_offset(zone_id)
|
|
659
|
+
value_array.append(format_iso_datetime_from_epoch_micros(epoch_micros, tz=row_zone))
|
|
660
|
+
except Exception as e:
|
|
661
|
+
_logger.error("Failed to parse TIMESTAMP_TZ row=%s: %s", row, e)
|
|
662
|
+
value_array.append('Failed to parse.')
|
|
646
663
|
elif d_type == VectorType.DECIMAL128:
|
|
647
664
|
# Handle both constant and non-constant vectors following Java implementation
|
|
648
665
|
if vector.isConstantVector:
|
|
@@ -679,6 +696,12 @@ def get_column_from_chunk(vector: Vector) -> list:
|
|
|
679
696
|
else:
|
|
680
697
|
value_array.append(None)
|
|
681
698
|
except Exception as e:
|
|
682
|
-
|
|
683
|
-
value_array
|
|
699
|
+
# Safety net: if anything escapes the per-row try/excepts above (or
|
|
700
|
+
# comes from a branch without one), pad value_array to vector.size so
|
|
701
|
+
# read_rows_from_chunk's columns[colIndex][rowIndex] access can never
|
|
702
|
+
# IndexError on a short column.
|
|
703
|
+
_logger.error("get_column_from_chunk failed (vectorType=%s, parsed=%s/%s): %s",
|
|
704
|
+
d_type, len(value_array), vector.size, e)
|
|
705
|
+
while len(value_array) < vector.size:
|
|
706
|
+
value_array.append('Failed to parse.')
|
|
684
707
|
return value_array
|
|
@@ -160,6 +160,177 @@ def floor_mod(x, y):
|
|
|
160
160
|
return r
|
|
161
161
|
|
|
162
162
|
|
|
163
|
+
_SECONDS_PER_DAY = 86400
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _civil_from_days(z):
|
|
167
|
+
"""
|
|
168
|
+
Convert days since 1970-01-01 (UTC) to proleptic Gregorian (year, month, day).
|
|
169
|
+
|
|
170
|
+
Howard Hinnant's algorithm
|
|
171
|
+
(http://howardhinnant.github.io/date_algorithms.html#civil_from_days).
|
|
172
|
+
Pure-integer math: works for any year representable by a Python int, so it
|
|
173
|
+
does not overflow like datetime.fromtimestamp (capped at year 9999). The
|
|
174
|
+
e6data engine emits dates that decode to years up to ~10^9 (matching Java's
|
|
175
|
+
LocalDate range); this is the only conversion path that can represent them.
|
|
176
|
+
|
|
177
|
+
Args:
|
|
178
|
+
z (int): Days since 1970-01-01 (negative = before epoch).
|
|
179
|
+
|
|
180
|
+
Returns:
|
|
181
|
+
tuple[int, int, int]: (year, month, day) in the proleptic Gregorian calendar.
|
|
182
|
+
"""
|
|
183
|
+
z = z + 719468
|
|
184
|
+
era = (z if z >= 0 else z - 146096) // 146097
|
|
185
|
+
doe = z - era * 146097
|
|
186
|
+
yoe = (doe - doe // 1460 + doe // 36524 - doe // 146096) // 365
|
|
187
|
+
y = yoe + era * 400
|
|
188
|
+
doy = doe - (365 * yoe + yoe // 4 - yoe // 100)
|
|
189
|
+
mp = (5 * doy + 2) // 153
|
|
190
|
+
d = doy - (153 * mp + 2) // 5 + 1
|
|
191
|
+
m = mp + 3 if mp < 10 else mp - 9
|
|
192
|
+
if m <= 2:
|
|
193
|
+
y += 1
|
|
194
|
+
return y, m, d
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _format_iso_year(year):
|
|
198
|
+
"""
|
|
199
|
+
Render the year component of an ISO-8601 date the way Java's
|
|
200
|
+
DateTimeFormatter.ISO_LOCAL_DATE does, so connector output matches the
|
|
201
|
+
e6data JDBC driver byte-for-byte:
|
|
202
|
+
|
|
203
|
+
- 0 <= year <= 9999 -> "YYYY" (zero-padded, no prefix)
|
|
204
|
+
- year > 9999 -> "+YYYYY..." (variable width, '+' prefix)
|
|
205
|
+
- year < 0 -> "-YYYY..." (zero-padded if 4 digits, '-' prefix)
|
|
206
|
+
"""
|
|
207
|
+
if 0 <= year <= 9999:
|
|
208
|
+
return "{:04d}".format(year)
|
|
209
|
+
if year > 9999:
|
|
210
|
+
return "+{}".format(year)
|
|
211
|
+
return "-{:04d}".format(abs(year)) if abs(year) <= 9999 else "-{}".format(abs(year))
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _tz_offset_minutes(tz, ref_seconds=0):
|
|
215
|
+
"""
|
|
216
|
+
Best-effort extraction of a UTC offset (in whole minutes) from a pytz tzinfo.
|
|
217
|
+
|
|
218
|
+
Used by the datetime formatter to apply the zone numerically, instead of
|
|
219
|
+
constructing a Python datetime (which can't represent years > 9999). Tries
|
|
220
|
+
tz.utcoffset(None) first — works directly for pytz.UTC and pytz.FixedOffset,
|
|
221
|
+
which is what timezone_from_offset() actually returns in practice. Falls
|
|
222
|
+
back to probing tz.utcoffset(<reference datetime>) for named zones, with
|
|
223
|
+
the probe seconds clamped to [0, year-9999] so it never hits the Python
|
|
224
|
+
datetime ceiling.
|
|
225
|
+
|
|
226
|
+
Args:
|
|
227
|
+
tz: A tzinfo / pytz timezone object, or None.
|
|
228
|
+
ref_seconds (int): Epoch seconds used to construct the probe datetime
|
|
229
|
+
for named zones. Ignored for fixed-offset / UTC.
|
|
230
|
+
|
|
231
|
+
Returns:
|
|
232
|
+
int: Offset minutes east of UTC. 0 when tz is None or cannot be probed.
|
|
233
|
+
"""
|
|
234
|
+
if tz is None:
|
|
235
|
+
return 0
|
|
236
|
+
try:
|
|
237
|
+
offset = tz.utcoffset(None)
|
|
238
|
+
if offset is not None:
|
|
239
|
+
return int(offset.total_seconds() // 60)
|
|
240
|
+
except (TypeError, AttributeError, ValueError):
|
|
241
|
+
pass
|
|
242
|
+
# Fallback for named timezones: probe with a reference datetime.
|
|
243
|
+
try:
|
|
244
|
+
from datetime import datetime as _dt
|
|
245
|
+
probe = _dt.utcfromtimestamp(max(0, min(ref_seconds, 253370764800))) # clamp to year 9999
|
|
246
|
+
offset = tz.utcoffset(probe)
|
|
247
|
+
if offset is not None:
|
|
248
|
+
return int(offset.total_seconds() // 60)
|
|
249
|
+
except Exception:
|
|
250
|
+
pass
|
|
251
|
+
return 0
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def format_iso_date_from_epoch_micros(epoch_micros):
|
|
255
|
+
"""
|
|
256
|
+
Format epoch microseconds (UTC) as an ISO-8601 date string with Java
|
|
257
|
+
LocalDate-style expanded-year support.
|
|
258
|
+
|
|
259
|
+
Pipeline: micros -> seconds -> days -> (year, month, day) -> string.
|
|
260
|
+
All steps are pure-int, so this never raises ValueError on year overflow
|
|
261
|
+
the way datetime.fromtimestamp(...).strftime(...) does for years > 9999.
|
|
262
|
+
|
|
263
|
+
Examples:
|
|
264
|
+
format_iso_date_from_epoch_micros(1770854400000000) -> "2026-02-12"
|
|
265
|
+
format_iso_date_from_epoch_micros(308847859200000000) -> "+11756-12-31"
|
|
266
|
+
format_iso_date_from_epoch_micros(0) -> "1970-01-01"
|
|
267
|
+
|
|
268
|
+
Args:
|
|
269
|
+
epoch_micros (int): Microseconds since 1970-01-01T00:00:00Z (may be negative).
|
|
270
|
+
|
|
271
|
+
Returns:
|
|
272
|
+
str: ISO-8601 date. Years 0-9999 unprefixed, others with '+' / '-'.
|
|
273
|
+
"""
|
|
274
|
+
epoch_seconds = floor_div(epoch_micros, 1_000_000)
|
|
275
|
+
days = floor_div(epoch_seconds, _SECONDS_PER_DAY)
|
|
276
|
+
y, mo, d = _civil_from_days(days)
|
|
277
|
+
return "{}-{:02d}-{:02d}".format(_format_iso_year(y), mo, d)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def format_iso_datetime_from_epoch_micros(epoch_micros, tz=None, separator='T',
|
|
281
|
+
include_millis=True, include_offset=True):
|
|
282
|
+
"""
|
|
283
|
+
Format epoch microseconds as an ISO-8601 datetime string with Java
|
|
284
|
+
LocalDateTime / OffsetDateTime-style expanded-year support.
|
|
285
|
+
|
|
286
|
+
The zone is applied numerically via _tz_offset_minutes(), so we never need
|
|
287
|
+
to construct a Python datetime — that's what allows the formatter to handle
|
|
288
|
+
years outside [1, 9999] without raising.
|
|
289
|
+
|
|
290
|
+
Single function with knobs to cover the three format variants the connector
|
|
291
|
+
historically emitted from datetime.strftime / datetime.isoformat:
|
|
292
|
+
|
|
293
|
+
- Row-wise DATETIME (read_values_from_array):
|
|
294
|
+
separator=' ', include_millis=False, include_offset=False
|
|
295
|
+
-> "2026-02-12 00:00:00"
|
|
296
|
+
|
|
297
|
+
- Chunk-wise DATETIME / TIMESTAMP_TZ (get_column_from_chunk), defaults:
|
|
298
|
+
separator='T', include_millis=True, include_offset=True
|
|
299
|
+
-> "2026-02-12T00:00:00.123+00:00"
|
|
300
|
+
-> "+11756-12-31T00:00:00.000+00:00" (year-overflow case)
|
|
301
|
+
|
|
302
|
+
Args:
|
|
303
|
+
epoch_micros (int): Microseconds since 1970-01-01T00:00:00Z.
|
|
304
|
+
tz: pytz timezone (UTC / FixedOffset / named); None means UTC.
|
|
305
|
+
separator (str): Between date and time (typically 'T' or ' ').
|
|
306
|
+
include_millis (bool): Append '.sss' from sub-second microseconds.
|
|
307
|
+
include_offset (bool): Append '+HH:MM' / '-HH:MM' UTC offset suffix.
|
|
308
|
+
|
|
309
|
+
Returns:
|
|
310
|
+
str: ISO-8601 datetime string.
|
|
311
|
+
"""
|
|
312
|
+
epoch_seconds_utc = floor_div(epoch_micros, 1_000_000)
|
|
313
|
+
offset_minutes = _tz_offset_minutes(tz, ref_seconds=epoch_seconds_utc)
|
|
314
|
+
local_micros = epoch_micros + offset_minutes * 60 * 1_000_000
|
|
315
|
+
epoch_seconds = floor_div(local_micros, 1_000_000)
|
|
316
|
+
micros_of_second = floor_mod(local_micros, 1_000_000)
|
|
317
|
+
days = floor_div(epoch_seconds, _SECONDS_PER_DAY)
|
|
318
|
+
sec_of_day = floor_mod(epoch_seconds, _SECONDS_PER_DAY)
|
|
319
|
+
y, mo, d = _civil_from_days(days)
|
|
320
|
+
h = sec_of_day // 3600
|
|
321
|
+
mi = (sec_of_day % 3600) // 60
|
|
322
|
+
s = sec_of_day % 60
|
|
323
|
+
out = "{}-{:02d}-{:02d}{}{:02d}:{:02d}:{:02d}".format(
|
|
324
|
+
_format_iso_year(y), mo, d, separator, h, mi, s)
|
|
325
|
+
if include_millis:
|
|
326
|
+
out += ".{:03d}".format(micros_of_second // 1000)
|
|
327
|
+
if include_offset:
|
|
328
|
+
sign = '+' if offset_minutes >= 0 else '-'
|
|
329
|
+
tot = abs(offset_minutes)
|
|
330
|
+
out += "{}{:02d}:{:02d}".format(sign, tot // 60, tot % 60)
|
|
331
|
+
return out
|
|
332
|
+
|
|
333
|
+
|
|
163
334
|
def timezone_from_offset(offset_string) -> pytz:
|
|
164
335
|
# Parse the offset string into hours and minutes
|
|
165
336
|
sign = -1 if offset_string[0] == "-" else 1
|