e6data-python-connector 2.2.1rc5__tar.gz → 2.2.2.3rc2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {e6data_python_connector-2.2.1rc5/e6data_python_connector.egg-info → e6data_python_connector-2.2.2.3rc2}/PKG-INFO +22 -11
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/README.md +1 -1
- e6data_python_connector-2.2.2.3rc2/e6data_python_connector/cluster_manager.py +261 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/e6data_grpc.py +40 -23
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2/e6data_python_connector.egg-info}/PKG-INFO +22 -11
- e6data_python_connector-2.2.2.3rc2/e6data_python_connector.egg-info/requires.txt +8 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/setup.py +27 -16
- e6data_python_connector-2.2.1rc5/e6data_python_connector/cluster_manager.py +0 -105
- e6data_python_connector-2.2.1rc5/e6data_python_connector.egg-info/requires.txt +0 -9
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/LICENSE +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/MANIFEST.in +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/__init__.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/cluster_server/__init__.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/cluster_server/cluster_pb2.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/cluster_server/cluster_pb2_grpc.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/common.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/constants.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/datainputstream.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/date_time_utils.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/dialect.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/e6x_vector/constants.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/e6x_vector/ttypes.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/exceptions.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/server/QueryEngineService.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/server/__init__.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/server/constants.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/server/e6x_engine_pb2.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/server/e6x_engine_pb2.pyi +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/server/e6x_engine_pb2_grpc.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/server/ttypes.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector/typeId.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector.egg-info/SOURCES.txt +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector.egg-info/dependency_links.txt +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector.egg-info/entry_points.txt +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/e6data_python_connector.egg-info/top_level.txt +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/pyproject.toml +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2.3rc2}/setup.cfg +0 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.2.3rc2
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
|
-
Author:
|
|
6
|
+
Author: e6data, Inc.
|
|
7
7
|
Author-email: info@e6data.com
|
|
8
8
|
License: Apache 2.0
|
|
9
9
|
Classifier: Operating System :: POSIX :: Linux
|
|
@@ -13,21 +13,32 @@ Classifier: Programming Language :: Python :: 3.6
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.7
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.8
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
19
|
Description-Content-Type: text/markdown
|
|
17
20
|
License-File: LICENSE
|
|
18
21
|
Requires-Dist: sqlalchemy>=1.0.0
|
|
19
|
-
Requires-Dist: future
|
|
20
|
-
Requires-Dist: python-dateutil
|
|
21
|
-
Requires-Dist: pycryptodome
|
|
22
|
-
Requires-Dist: pytz
|
|
23
|
-
Requires-Dist: thrift
|
|
24
|
-
Requires-Dist: grpcio
|
|
22
|
+
Requires-Dist: future==1.0.0
|
|
23
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
24
|
+
Requires-Dist: pycryptodome==3.19.1
|
|
25
|
+
Requires-Dist: pytz==2024.1
|
|
26
|
+
Requires-Dist: thrift==0.20.0
|
|
27
|
+
Requires-Dist: grpcio==1.65.1
|
|
25
28
|
Requires-Dist: grpcio-tools
|
|
26
|
-
|
|
29
|
+
Dynamic: author
|
|
30
|
+
Dynamic: author-email
|
|
31
|
+
Dynamic: classifier
|
|
32
|
+
Dynamic: description
|
|
33
|
+
Dynamic: description-content-type
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: summary
|
|
27
38
|
|
|
28
39
|
# e6data Python Connector
|
|
29
40
|
|
|
30
|
-

|
|
31
42
|
|
|
32
43
|
## Introduction
|
|
33
44
|
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
import time
|
|
3
|
+
import e6data_python_connector.cluster_server.cluster_pb2 as cluster_pb2
|
|
4
|
+
import e6data_python_connector.cluster_server.cluster_pb2_grpc as cluster_pb2_grpc
|
|
5
|
+
import grpc
|
|
6
|
+
from grpc._channel import _InactiveRpcError
|
|
7
|
+
import multiprocessing
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _get_grpc_header(engine_ip=None, cluster=None):
|
|
11
|
+
metadata = []
|
|
12
|
+
if engine_ip:
|
|
13
|
+
metadata.append(('plannerip', engine_ip))
|
|
14
|
+
if cluster:
|
|
15
|
+
metadata.append(('cluster-uuid', cluster))
|
|
16
|
+
return metadata
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class _StatusLock:
|
|
20
|
+
"""
|
|
21
|
+
A thread-safe and process-safe lock manager designed for managing
|
|
22
|
+
concurrent access protection in multithreaded and multiprocessing environments.
|
|
23
|
+
|
|
24
|
+
This class encapsulates locking mechanisms using threading and multiprocessing
|
|
25
|
+
modules to ensure an operation's atomicity and handle shared resources safely.
|
|
26
|
+
|
|
27
|
+
Attributes:
|
|
28
|
+
_LOCK_TIMEOUT (int): The maximum timeout (in milliseconds) to try
|
|
29
|
+
acquiring a lock before raising an error.
|
|
30
|
+
_status_thread_lock (threading.Lock): A thread-level lock to synchronize
|
|
31
|
+
access among threads in the same process.
|
|
32
|
+
_status_multiprocessing_lock (multiprocessing.Semaphore): A
|
|
33
|
+
process-level lock to synchronize access across different processes.
|
|
34
|
+
_is_active (bool): A boolean flag indicating whether the lock
|
|
35
|
+
is currently active (True) or not (False).
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
_LOCK_TIMEOUT = 500
|
|
39
|
+
|
|
40
|
+
def __init__(self):
|
|
41
|
+
"""
|
|
42
|
+
Initializes the _StatusLock instance with its respective
|
|
43
|
+
thread and multiprocessing locks and sets the active flag to False.
|
|
44
|
+
"""
|
|
45
|
+
self._status_thread_lock = threading.Lock()
|
|
46
|
+
self._status_multiprocessing_lock = multiprocessing.Semaphore()
|
|
47
|
+
self._is_active = False
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def is_active(self) -> bool:
|
|
51
|
+
"""
|
|
52
|
+
Checks if the lock is currently active.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
bool: True if the lock is active, False otherwise.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
return self._is_active
|
|
59
|
+
|
|
60
|
+
def set_active(self):
|
|
61
|
+
"""
|
|
62
|
+
Activates the lock by setting the `_is_active` flag to True.
|
|
63
|
+
This can be used for manually marking the state of the lock
|
|
64
|
+
as active during synchronization operations.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
self._is_active = True
|
|
68
|
+
|
|
69
|
+
def __enter__(self):
|
|
70
|
+
"""
|
|
71
|
+
Enters a context-managed locking block.
|
|
72
|
+
|
|
73
|
+
Acquires both the thread-level lock and process-level semaphore
|
|
74
|
+
to ensure the current operation can safely access shared resources.
|
|
75
|
+
|
|
76
|
+
Raises:
|
|
77
|
+
TimeoutError: If the lock cannot be acquired within the timeout period.
|
|
78
|
+
|
|
79
|
+
Returns:
|
|
80
|
+
_StatusLock: The current instance of the lock, used for context management.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
self._status_thread_lock.acquire(timeout=self._LOCK_TIMEOUT)
|
|
84
|
+
self._status_multiprocessing_lock.acquire(timeout=self._LOCK_TIMEOUT)
|
|
85
|
+
return self
|
|
86
|
+
|
|
87
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
88
|
+
"""
|
|
89
|
+
Exits a context-managed locking block.
|
|
90
|
+
|
|
91
|
+
Releases both the thread-level lock and process-level semaphore
|
|
92
|
+
to allow other operations to acquire the lock. Handles cleanup
|
|
93
|
+
regardless of whether an exception occurred.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
exc_type (Type[BaseException]): The type of exception raised (if any).
|
|
97
|
+
exc_val (BaseException): The exception instance raised (if any).
|
|
98
|
+
exc_tb (Traceback): The traceback object of the exception (if any).
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
self._status_thread_lock.release()
|
|
102
|
+
self._status_multiprocessing_lock.release()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
status_lock = _StatusLock()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class ClusterManager:
|
|
109
|
+
"""
|
|
110
|
+
A manager for handling operations and connections with a remote cluster infrastructure.
|
|
111
|
+
Provides methods to manage cluster states, such as resuming or suspending its activity,
|
|
112
|
+
by interacting with a gRPC-based remote service.
|
|
113
|
+
|
|
114
|
+
Attributes:
|
|
115
|
+
_host (str): The hostname or IP address of the cluster service.
|
|
116
|
+
_port (int): The port number used to connect to the cluster service.
|
|
117
|
+
_user (str): The username for authentication with the cluster service.
|
|
118
|
+
_password (str): The password for authentication with the cluster service.
|
|
119
|
+
_timeout (float): The timeout threshold (as an epoch timestamp) for operations,
|
|
120
|
+
defaulting to 3 minutes from initialization.
|
|
121
|
+
_secure_channel (bool): Indicates whether a secure gRPC channel
|
|
122
|
+
(SSL/TLS) should be used for communication; defaults to False.
|
|
123
|
+
cluster_uuid (str): The unique identifier for the target cluster.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 3, cluster_uuid=None):
|
|
127
|
+
"""
|
|
128
|
+
Initializes a new instance of the ClusterManager class.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
host (str): The hostname or IP address of the cluster service.
|
|
132
|
+
port (int): The port number for accessing the cluster service.
|
|
133
|
+
user (str): The username used for connecting to the cluster service.
|
|
134
|
+
password (str): The password used for connecting to the cluster service.
|
|
135
|
+
secure_channel (bool, optional): Whether to use a secure
|
|
136
|
+
gRPC channel for communication; defaults to False.
|
|
137
|
+
timeout (int, optional): The timeout duration (in seconds) for operations;
|
|
138
|
+
defaults to 3 minutes.
|
|
139
|
+
cluster_uuid (str, optional): The unique identifier for the target cluster;
|
|
140
|
+
defaults to None.
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
self._host = host
|
|
144
|
+
self._port = port
|
|
145
|
+
self._user = user
|
|
146
|
+
self._password = password
|
|
147
|
+
self._timeout = time.time() + timeout
|
|
148
|
+
self._secure_channel = secure_channel
|
|
149
|
+
self.cluster_uuid = cluster_uuid
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def _get_connection(self):
|
|
153
|
+
"""
|
|
154
|
+
Dynamically establishes a gRPC connection to the cluster service.
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
cluster_pb2_grpc.ClusterServiceStub: A gRPC client stub
|
|
158
|
+
for interacting with the cluster service methods.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
if self._secure_channel:
|
|
162
|
+
self._channel = grpc.secure_channel(
|
|
163
|
+
target='{}:{}'.format(self._host, self._port),
|
|
164
|
+
credentials=grpc.ssl_channel_credentials()
|
|
165
|
+
)
|
|
166
|
+
else:
|
|
167
|
+
self._channel = grpc.insecure_channel(
|
|
168
|
+
target='{}:{}'.format(self._host, self._port)
|
|
169
|
+
)
|
|
170
|
+
return cluster_pb2_grpc.ClusterServiceStub(self._channel)
|
|
171
|
+
|
|
172
|
+
def resume(self) -> bool:
|
|
173
|
+
"""
|
|
174
|
+
Resumes the cluster if it is currently suspended or not in the 'active' state.
|
|
175
|
+
|
|
176
|
+
This method interacts with the remote cluster service to verify its current
|
|
177
|
+
status. If suspended, it sends a resume request and monitors
|
|
178
|
+
the cluster's state until it becomes active or fails.
|
|
179
|
+
|
|
180
|
+
The operation uses a locking mechanism (`status_lock`) to ensure thread-safe
|
|
181
|
+
and process-safe state transitions.
|
|
182
|
+
|
|
183
|
+
Returns:
|
|
184
|
+
bool: True if the cluster resumes successfully and becomes active;
|
|
185
|
+
False if the cluster cannot be resumed, remains suspended, or fails.
|
|
186
|
+
|
|
187
|
+
Raises:
|
|
188
|
+
_InactiveRpcError: An exception raised if there is a communication error
|
|
189
|
+
while interacting with the remote cluster service.
|
|
190
|
+
|
|
191
|
+
Notes:
|
|
192
|
+
- If the cluster is already active, this method completes successfully
|
|
193
|
+
without further actions.
|
|
194
|
+
- If the cluster is in a 'resuming' state, this method waits for the
|
|
195
|
+
cluster to transition to 'active' or any terminal state (e.g., 'failed').
|
|
196
|
+
- The operation is subject to the `_timeout` threshold;
|
|
197
|
+
if the timeout expires, the method returns False.
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
with status_lock as lock:
|
|
201
|
+
if lock.is_active:
|
|
202
|
+
return True
|
|
203
|
+
|
|
204
|
+
# Retrieve the current cluster status
|
|
205
|
+
status_payload = cluster_pb2.ClusterStatusRequest(
|
|
206
|
+
user=self._user,
|
|
207
|
+
password=self._password
|
|
208
|
+
)
|
|
209
|
+
current_status = self._get_connection.status(
|
|
210
|
+
status_payload,
|
|
211
|
+
metadata=_get_grpc_header(cluster=self.cluster_uuid)
|
|
212
|
+
)
|
|
213
|
+
if current_status.status == 'suspended':
|
|
214
|
+
# Send the resume request
|
|
215
|
+
payload = cluster_pb2.ResumeRequest(
|
|
216
|
+
user=self._user,
|
|
217
|
+
password=self._password
|
|
218
|
+
)
|
|
219
|
+
response = self._get_connection.resume(
|
|
220
|
+
payload,
|
|
221
|
+
metadata=_get_grpc_header(cluster=self.cluster_uuid)
|
|
222
|
+
)
|
|
223
|
+
elif current_status.status == 'active':
|
|
224
|
+
return True
|
|
225
|
+
elif current_status.status != 'resuming':
|
|
226
|
+
"""
|
|
227
|
+
If cluster cannot be resumed due to its current state,
|
|
228
|
+
or already in a process of resuming, terminate the operation.
|
|
229
|
+
"""
|
|
230
|
+
return False
|
|
231
|
+
|
|
232
|
+
# Wait for the cluster to become active
|
|
233
|
+
while True:
|
|
234
|
+
try:
|
|
235
|
+
status_payload = cluster_pb2.ClusterStatusRequest(
|
|
236
|
+
user=self._user,
|
|
237
|
+
password=self._password
|
|
238
|
+
)
|
|
239
|
+
response = self._get_connection.status(
|
|
240
|
+
status_payload,
|
|
241
|
+
metadata=_get_grpc_header(cluster=self.cluster_uuid)
|
|
242
|
+
)
|
|
243
|
+
if response.status == 'active':
|
|
244
|
+
lock.set_active()
|
|
245
|
+
return True
|
|
246
|
+
if response.status in ['suspended', 'failed']:
|
|
247
|
+
return False
|
|
248
|
+
if time.time() > self._timeout:
|
|
249
|
+
return False
|
|
250
|
+
except _InactiveRpcError as e:
|
|
251
|
+
pass
|
|
252
|
+
time.sleep(5)
|
|
253
|
+
|
|
254
|
+
def suspend(self):
|
|
255
|
+
"""
|
|
256
|
+
Suspends the cluster operations (not implemented).
|
|
257
|
+
|
|
258
|
+
Placeholder method to be implemented for suspending a cluster,
|
|
259
|
+
typically interacting with the remote cluster service API.
|
|
260
|
+
"""
|
|
261
|
+
pass
|
|
@@ -11,6 +11,7 @@ import datetime
|
|
|
11
11
|
import logging
|
|
12
12
|
import re
|
|
13
13
|
import sys
|
|
14
|
+
import time
|
|
14
15
|
from decimal import Decimal
|
|
15
16
|
from io import BytesIO
|
|
16
17
|
from ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
|
|
@@ -53,14 +54,38 @@ def _parse_timestamp(value):
|
|
|
53
54
|
value = datetime.datetime.strptime(value, format)
|
|
54
55
|
else:
|
|
55
56
|
raise Exception(
|
|
56
|
-
'Cannot convert "{}" into a datetime'.format(value)
|
|
57
|
+
'Cannot convert "{}" into a datetime'.format(value)
|
|
58
|
+
)
|
|
57
59
|
else:
|
|
58
60
|
value = None
|
|
59
61
|
return value
|
|
60
62
|
|
|
61
63
|
|
|
62
|
-
TYPES_CONVERTER = {
|
|
63
|
-
|
|
64
|
+
TYPES_CONVERTER = {
|
|
65
|
+
"DECIMAL_TYPE": Decimal,
|
|
66
|
+
"TIMESTAMP_TYPE": _parse_timestamp
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def re_auth(func):
|
|
71
|
+
def wrapper(self, *args, **kwargs):
|
|
72
|
+
max_retry = 5
|
|
73
|
+
current_retry = 0
|
|
74
|
+
while current_retry < max_retry:
|
|
75
|
+
try:
|
|
76
|
+
return func(self, *args, **kwargs)
|
|
77
|
+
except _InactiveRpcError as e:
|
|
78
|
+
current_retry += 1
|
|
79
|
+
if current_retry == max_retry:
|
|
80
|
+
raise e
|
|
81
|
+
if e.code() == grpc.StatusCode.INTERNAL and 'Access denied' in e.details():
|
|
82
|
+
time.sleep(0.2)
|
|
83
|
+
print(f'RE_AUTH: Function Name: {func}')
|
|
84
|
+
print(f'RE_AUTH: Error Found {e}')
|
|
85
|
+
else:
|
|
86
|
+
raise e
|
|
87
|
+
|
|
88
|
+
return wrapper
|
|
64
89
|
|
|
65
90
|
|
|
66
91
|
class HiveParamEscaper(ParamEscaper):
|
|
@@ -148,8 +173,6 @@ class Connection(object):
|
|
|
148
173
|
- max_receive_message_length: This parameter sets the maximum allowed size (in bytes) for incoming messages on the gRPC server.
|
|
149
174
|
- max_send_message_length: Similar to max_receive_message_length, this parameter sets the maximum allowed size (in bytes) for outgoing messages from the gRPC client
|
|
150
175
|
- grpc_prepare_timeout: Timeout for prepare statement API call (default to 10 minutes).
|
|
151
|
-
#keep-alive parameters
|
|
152
|
-
- keepalive_time_ms: This parameter defines the time, in milliseconds, Default to 30 seconds
|
|
153
176
|
"""
|
|
154
177
|
if not username or not password:
|
|
155
178
|
raise ValueError("username or password cannot be empty.")
|
|
@@ -170,18 +193,16 @@ class Connection(object):
|
|
|
170
193
|
self._auto_resume = auto_resume
|
|
171
194
|
|
|
172
195
|
self._keepalive_timeout_ms = 900000
|
|
173
|
-
self._keepalive_time_ms = 30000
|
|
174
196
|
self._max_receive_message_length = -1
|
|
175
197
|
self._max_send_message_length = 300 * 1024 * 1024 # mb
|
|
176
198
|
self.grpc_prepare_timeout = 10 * 60 # 10 minutes
|
|
177
199
|
|
|
178
|
-
if
|
|
200
|
+
if isinstance(grpc_options, dict):
|
|
179
201
|
self._keepalive_timeout_ms = grpc_options.get('keepalive_timeout_ms') or self._keepalive_timeout_ms
|
|
180
202
|
self._max_receive_message_length = grpc_options.get(
|
|
181
203
|
'max_receive_message_length') or self._max_receive_message_length
|
|
182
204
|
self._max_send_message_length = grpc_options.get('max_send_message_length') or self._max_send_message_length
|
|
183
205
|
self.grpc_prepare_timeout = grpc_options.get('grpc_prepare_timeout') or self.grpc_prepare_timeout
|
|
184
|
-
self._keepalive_time_ms = grpc_options.get('keepalive_time_ms') or self._keepalive_time_ms
|
|
185
206
|
self._create_client()
|
|
186
207
|
|
|
187
208
|
def _create_client(self):
|
|
@@ -191,12 +212,7 @@ class Connection(object):
|
|
|
191
212
|
options=[
|
|
192
213
|
("grpc.keepalive_timeout_ms", self._keepalive_timeout_ms),
|
|
193
214
|
('grpc.max_send_message_length', self._max_send_message_length),
|
|
194
|
-
('grpc.max_receive_message_length', self._max_receive_message_length)
|
|
195
|
-
('grpc.keepalive_time_ms', self._keepalive_time_ms),
|
|
196
|
-
('grpc.keepalive_permit_without_calls', 1), # Allow keep-alives with no active RPCs
|
|
197
|
-
('grpc.http2.max_pings_without_data', 0), # Unlimited pings without data
|
|
198
|
-
('grpc.http2.min_time_between_pings_ms', 15000), # Minimum 15 seconds between pings
|
|
199
|
-
('grpc.http2.min_ping_interval_without_data_ms', 15000)# Minimum 15 seconds between pings without data
|
|
215
|
+
('grpc.max_receive_message_length', self._max_receive_message_length)
|
|
200
216
|
],
|
|
201
217
|
credentials=grpc.ssl_channel_credentials()
|
|
202
218
|
)
|
|
@@ -206,16 +222,15 @@ class Connection(object):
|
|
|
206
222
|
options=[
|
|
207
223
|
("grpc.keepalive_timeout_ms", self._keepalive_timeout_ms),
|
|
208
224
|
('grpc.max_send_message_length', self._max_send_message_length),
|
|
209
|
-
('grpc.max_receive_message_length', self._max_receive_message_length)
|
|
210
|
-
('grpc.keepalive_time_ms', self._keepalive_time_ms),
|
|
211
|
-
('grpc.keepalive_permit_without_calls', 1), # Allow keep-alives with no active RPCs
|
|
212
|
-
('grpc.http2.max_pings_without_data', 0), # Unlimited pings without data
|
|
213
|
-
('grpc.http2.min_time_between_pings_ms', 15000), # Minimum 15 seconds between pings
|
|
214
|
-
('grpc.http2.min_ping_interval_without_data_ms', 15000)# Minimum 15 seconds between pings without data
|
|
225
|
+
('grpc.max_receive_message_length', self._max_receive_message_length)
|
|
215
226
|
]
|
|
216
227
|
)
|
|
217
228
|
self._client = e6x_engine_pb2_grpc.QueryEngineServiceStub(self._channel)
|
|
218
229
|
|
|
230
|
+
def get_re_authenticate_session_id(self):
|
|
231
|
+
self._session_id = None
|
|
232
|
+
return self.get_session_id
|
|
233
|
+
|
|
219
234
|
@property
|
|
220
235
|
def get_session_id(self):
|
|
221
236
|
"""
|
|
@@ -236,13 +251,14 @@ class Connection(object):
|
|
|
236
251
|
raise ValueError("Invalid credentials.")
|
|
237
252
|
except _InactiveRpcError as e:
|
|
238
253
|
if self._auto_resume:
|
|
239
|
-
if e.code() == grpc.StatusCode.
|
|
254
|
+
if e.code() == grpc.StatusCode.UNAVAILABLE and 'status: 503' in e.details():
|
|
240
255
|
status = ClusterManager(
|
|
241
256
|
host=self._host,
|
|
242
257
|
port=self._port,
|
|
243
258
|
user=self.__username,
|
|
244
259
|
password=self.__password,
|
|
245
|
-
secure_channel=self._secure_channel
|
|
260
|
+
secure_channel=self._secure_channel,
|
|
261
|
+
cluster_uuid=self.cluster_uuid
|
|
246
262
|
).resume()
|
|
247
263
|
if status:
|
|
248
264
|
authenticate_request = e6x_engine_pb2.AuthenticateRequest(
|
|
@@ -510,6 +526,7 @@ class Cursor(DBAPICursor):
|
|
|
510
526
|
)
|
|
511
527
|
return self.connection.client.status(status_request, metadata=self.metadata)
|
|
512
528
|
|
|
529
|
+
@re_auth
|
|
513
530
|
def execute(self, operation, parameters=None, **kwargs):
|
|
514
531
|
"""Prepare and execute a database operation (query or command).
|
|
515
532
|
Return values are not defined.
|
|
@@ -565,6 +582,7 @@ class Cursor(DBAPICursor):
|
|
|
565
582
|
|
|
566
583
|
self._query_id = prepare_statement_response.queryId
|
|
567
584
|
self._engine_ip = prepare_statement_response.engineIP
|
|
585
|
+
|
|
568
586
|
execute_statement_request = e6x_engine_pb2.ExecuteStatementV2Request(
|
|
569
587
|
engineIP=self._engine_ip,
|
|
570
588
|
sessionId=self.connection.get_session_id,
|
|
@@ -649,7 +667,6 @@ class Cursor(DBAPICursor):
|
|
|
649
667
|
return self._fetch_all()
|
|
650
668
|
|
|
651
669
|
def fetchmany(self, size: int = None):
|
|
652
|
-
# _logger.info("fetching all from overriden method")
|
|
653
670
|
if size is None:
|
|
654
671
|
size = self.arraysize
|
|
655
672
|
if self._data is None:
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.2.3rc2
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
|
-
Author:
|
|
6
|
+
Author: e6data, Inc.
|
|
7
7
|
Author-email: info@e6data.com
|
|
8
8
|
License: Apache 2.0
|
|
9
9
|
Classifier: Operating System :: POSIX :: Linux
|
|
@@ -13,21 +13,32 @@ Classifier: Programming Language :: Python :: 3.6
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.7
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.8
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
19
|
Description-Content-Type: text/markdown
|
|
17
20
|
License-File: LICENSE
|
|
18
21
|
Requires-Dist: sqlalchemy>=1.0.0
|
|
19
|
-
Requires-Dist: future
|
|
20
|
-
Requires-Dist: python-dateutil
|
|
21
|
-
Requires-Dist: pycryptodome
|
|
22
|
-
Requires-Dist: pytz
|
|
23
|
-
Requires-Dist: thrift
|
|
24
|
-
Requires-Dist: grpcio
|
|
22
|
+
Requires-Dist: future==1.0.0
|
|
23
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
24
|
+
Requires-Dist: pycryptodome==3.19.1
|
|
25
|
+
Requires-Dist: pytz==2024.1
|
|
26
|
+
Requires-Dist: thrift==0.20.0
|
|
27
|
+
Requires-Dist: grpcio==1.65.1
|
|
25
28
|
Requires-Dist: grpcio-tools
|
|
26
|
-
|
|
29
|
+
Dynamic: author
|
|
30
|
+
Dynamic: author-email
|
|
31
|
+
Dynamic: classifier
|
|
32
|
+
Dynamic: description
|
|
33
|
+
Dynamic: description-content-type
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: summary
|
|
27
38
|
|
|
28
39
|
# e6data Python Connector
|
|
29
40
|
|
|
30
|
-

|
|
31
42
|
|
|
32
43
|
## Introduction
|
|
33
44
|
|
|
@@ -12,37 +12,45 @@
|
|
|
12
12
|
|
|
13
13
|
import setuptools
|
|
14
14
|
|
|
15
|
-
VERSION =
|
|
15
|
+
VERSION = (2, 2, 2, '3.rc2')
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
def get_long_desc():
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
"""
|
|
20
|
+
Reads the `README.md` file to provide a detailed description of the package.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
str: The content of the README.md file as a string.
|
|
24
|
+
"""
|
|
25
|
+
try:
|
|
26
|
+
with open("README.md", "r", encoding="utf-8") as fh: # Explicitly specifying encoding.
|
|
27
|
+
return fh.read()
|
|
28
|
+
except FileNotFoundError:
|
|
29
|
+
# Fallback if README.md is missing.
|
|
30
|
+
return "Client for the e6data distributed SQL Engine."
|
|
22
31
|
|
|
23
32
|
|
|
24
33
|
setuptools.setup(
|
|
25
34
|
name="e6data-python-connector",
|
|
26
|
-
version=
|
|
27
|
-
author="
|
|
35
|
+
version=".".join(map(str, VERSION)), # More Pythonic way to convert tuple to string.
|
|
36
|
+
author="e6data, Inc.",
|
|
28
37
|
author_email="info@e6data.com",
|
|
29
38
|
description="Client for the e6data distributed SQL Engine.",
|
|
30
39
|
long_description=get_long_desc(),
|
|
31
40
|
long_description_content_type="text/markdown",
|
|
32
41
|
url='https://github.com/e6x-labs/e6data-python-connector',
|
|
33
|
-
packages=setuptools.find_packages(),
|
|
42
|
+
packages=setuptools.find_packages(), # Automatically detects all packages in the source tree.
|
|
34
43
|
license="Apache 2.0",
|
|
35
|
-
include_package_data=True,
|
|
44
|
+
include_package_data=True, # Includes non-Python files specified in MANIFEST.in.
|
|
36
45
|
install_requires=[
|
|
37
46
|
'sqlalchemy>=1.0.0',
|
|
38
|
-
'future',
|
|
39
|
-
'python-dateutil',
|
|
40
|
-
'pycryptodome',
|
|
41
|
-
'pytz',
|
|
42
|
-
'thrift',
|
|
43
|
-
'grpcio',
|
|
47
|
+
'future==1.0.0',
|
|
48
|
+
'python-dateutil==2.9.0.post0',
|
|
49
|
+
'pycryptodome==3.19.1',
|
|
50
|
+
'pytz==2024.1',
|
|
51
|
+
'thrift==0.20.0',
|
|
52
|
+
'grpcio==1.65.1',
|
|
44
53
|
'grpcio-tools',
|
|
45
|
-
'sqlalchemy'
|
|
46
54
|
],
|
|
47
55
|
classifiers=[
|
|
48
56
|
"Operating System :: POSIX :: Linux",
|
|
@@ -51,7 +59,10 @@ setuptools.setup(
|
|
|
51
59
|
'Programming Language :: Python :: 3.6',
|
|
52
60
|
'Programming Language :: Python :: 3.7',
|
|
53
61
|
'Programming Language :: Python :: 3.8',
|
|
54
|
-
'Programming Language :: Python :: 3.9'
|
|
62
|
+
'Programming Language :: Python :: 3.9',
|
|
63
|
+
"Programming Language :: Python :: 3.10",
|
|
64
|
+
"Programming Language :: Python :: 3.11",
|
|
65
|
+
"Programming Language :: Python :: 3.12",
|
|
55
66
|
],
|
|
56
67
|
entry_points={
|
|
57
68
|
'sqlalchemy.dialects': [
|
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
import threading
|
|
2
|
-
import time
|
|
3
|
-
import e6data_python_connector.cluster_server.cluster_pb2 as cluster_pb2
|
|
4
|
-
import e6data_python_connector.cluster_server.cluster_pb2_grpc as cluster_pb2_grpc
|
|
5
|
-
import grpc
|
|
6
|
-
from grpc._channel import _InactiveRpcError
|
|
7
|
-
import multiprocessing
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class _StatusLock:
|
|
11
|
-
_LOCK_TIMEOUT = 500
|
|
12
|
-
|
|
13
|
-
def __init__(self):
|
|
14
|
-
self._status_thread_lock = threading.Lock()
|
|
15
|
-
self._status_multiprocessing_lock = multiprocessing.Semaphore()
|
|
16
|
-
self._is_active = False
|
|
17
|
-
|
|
18
|
-
@property
|
|
19
|
-
def is_active(self) -> bool:
|
|
20
|
-
return self._is_active
|
|
21
|
-
|
|
22
|
-
def set_active(self):
|
|
23
|
-
self._is_active = True
|
|
24
|
-
|
|
25
|
-
def __enter__(self):
|
|
26
|
-
self._status_thread_lock.acquire(timeout=self._LOCK_TIMEOUT)
|
|
27
|
-
self._status_multiprocessing_lock.acquire(timeout=self._LOCK_TIMEOUT)
|
|
28
|
-
return self
|
|
29
|
-
|
|
30
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
31
|
-
self._status_thread_lock.release()
|
|
32
|
-
self._status_multiprocessing_lock.release()
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
status_lock = _StatusLock()
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
class ClusterManager:
|
|
39
|
-
def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 3):
|
|
40
|
-
self._host = host
|
|
41
|
-
self._port = port
|
|
42
|
-
self._user = user
|
|
43
|
-
self._password = password
|
|
44
|
-
self._timeout = time.time() + timeout
|
|
45
|
-
self._secure_channel = secure_channel
|
|
46
|
-
|
|
47
|
-
@property
|
|
48
|
-
def _get_connection(self):
|
|
49
|
-
if self._secure_channel:
|
|
50
|
-
self._channel = grpc.secure_channel(
|
|
51
|
-
target='{}:{}'.format(self._host, self._port),
|
|
52
|
-
credentials=grpc.ssl_channel_credentials()
|
|
53
|
-
)
|
|
54
|
-
else:
|
|
55
|
-
self._channel = grpc.insecure_channel(
|
|
56
|
-
target='{}:{}'.format(self._host, self._port)
|
|
57
|
-
)
|
|
58
|
-
return cluster_pb2_grpc.ClusterServiceStub(self._channel)
|
|
59
|
-
|
|
60
|
-
def resume(self) -> bool:
|
|
61
|
-
"""
|
|
62
|
-
Return True if cluster came to active state else False.
|
|
63
|
-
"""
|
|
64
|
-
with status_lock as lock:
|
|
65
|
-
if lock.is_active:
|
|
66
|
-
return True
|
|
67
|
-
status_payload = cluster_pb2.ClusterStatusRequest(
|
|
68
|
-
user=self._user,
|
|
69
|
-
password=self._password
|
|
70
|
-
)
|
|
71
|
-
current_status = self._get_connection.status(status_payload)
|
|
72
|
-
if current_status.status == 'suspended':
|
|
73
|
-
payload = cluster_pb2.ResumeRequest(
|
|
74
|
-
user=self._user,
|
|
75
|
-
password=self._password
|
|
76
|
-
)
|
|
77
|
-
response = self._get_connection.resume(payload)
|
|
78
|
-
elif current_status.status == 'active':
|
|
79
|
-
return True
|
|
80
|
-
elif current_status.status != 'resuming':
|
|
81
|
-
"""
|
|
82
|
-
If cluster is in resuming state already, start watching for the status.
|
|
83
|
-
Cluster is in different state, cannot resume.
|
|
84
|
-
"""
|
|
85
|
-
return False
|
|
86
|
-
while True:
|
|
87
|
-
try:
|
|
88
|
-
status_payload = cluster_pb2.ClusterStatusRequest(
|
|
89
|
-
user=self._user,
|
|
90
|
-
password=self._password
|
|
91
|
-
)
|
|
92
|
-
response = self._get_connection.status(status_payload)
|
|
93
|
-
if response.status == 'active':
|
|
94
|
-
lock.set_active()
|
|
95
|
-
return True
|
|
96
|
-
if response.status in ['suspended', 'failed']:
|
|
97
|
-
return False
|
|
98
|
-
if time.time() > self._timeout:
|
|
99
|
-
return False
|
|
100
|
-
except _InactiveRpcError as e:
|
|
101
|
-
pass
|
|
102
|
-
time.sleep(5)
|
|
103
|
-
|
|
104
|
-
def suspend(self):
|
|
105
|
-
pass
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|