e6data-python-connector 2.2.1rc5__tar.gz → 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {e6data_python_connector-2.2.1rc5/e6data_python_connector.egg-info → e6data_python_connector-2.2.2}/PKG-INFO +22 -11
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/README.md +1 -1
- e6data_python_connector-2.2.2/e6data_python_connector/cluster_manager.py +261 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/e6data_grpc.py +36 -23
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2/e6data_python_connector.egg-info}/PKG-INFO +22 -11
- e6data_python_connector-2.2.2/e6data_python_connector.egg-info/requires.txt +8 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/setup.py +27 -16
- e6data_python_connector-2.2.1rc5/e6data_python_connector/cluster_manager.py +0 -105
- e6data_python_connector-2.2.1rc5/e6data_python_connector.egg-info/requires.txt +0 -9
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/LICENSE +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/MANIFEST.in +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/__init__.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/cluster_server/__init__.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/cluster_server/cluster_pb2.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/cluster_server/cluster_pb2_grpc.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/common.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/constants.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/datainputstream.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/date_time_utils.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/dialect.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/e6x_vector/constants.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/e6x_vector/ttypes.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/exceptions.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/server/QueryEngineService.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/server/__init__.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/server/constants.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/server/e6x_engine_pb2.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/server/e6x_engine_pb2.pyi +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/server/e6x_engine_pb2_grpc.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/server/ttypes.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/typeId.py +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector.egg-info/SOURCES.txt +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector.egg-info/dependency_links.txt +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector.egg-info/entry_points.txt +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector.egg-info/top_level.txt +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/pyproject.toml +0 -0
- {e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/setup.cfg +0 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.2
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
|
-
Author:
|
|
6
|
+
Author: e6data, Inc.
|
|
7
7
|
Author-email: info@e6data.com
|
|
8
8
|
License: Apache 2.0
|
|
9
9
|
Classifier: Operating System :: POSIX :: Linux
|
|
@@ -13,21 +13,32 @@ Classifier: Programming Language :: Python :: 3.6
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.7
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.8
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
19
|
Description-Content-Type: text/markdown
|
|
17
20
|
License-File: LICENSE
|
|
18
21
|
Requires-Dist: sqlalchemy>=1.0.0
|
|
19
|
-
Requires-Dist: future
|
|
20
|
-
Requires-Dist: python-dateutil
|
|
21
|
-
Requires-Dist: pycryptodome
|
|
22
|
-
Requires-Dist: pytz
|
|
23
|
-
Requires-Dist: thrift
|
|
24
|
-
Requires-Dist: grpcio
|
|
22
|
+
Requires-Dist: future==1.0.0
|
|
23
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
24
|
+
Requires-Dist: pycryptodome==3.19.1
|
|
25
|
+
Requires-Dist: pytz==2024.1
|
|
26
|
+
Requires-Dist: thrift==0.20.0
|
|
27
|
+
Requires-Dist: grpcio==1.65.1
|
|
25
28
|
Requires-Dist: grpcio-tools
|
|
26
|
-
|
|
29
|
+
Dynamic: author
|
|
30
|
+
Dynamic: author-email
|
|
31
|
+
Dynamic: classifier
|
|
32
|
+
Dynamic: description
|
|
33
|
+
Dynamic: description-content-type
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: summary
|
|
27
38
|
|
|
28
39
|
# e6data Python Connector
|
|
29
40
|
|
|
30
|
-

|
|
31
42
|
|
|
32
43
|
## Introduction
|
|
33
44
|
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
import time
|
|
3
|
+
import e6data_python_connector.cluster_server.cluster_pb2 as cluster_pb2
|
|
4
|
+
import e6data_python_connector.cluster_server.cluster_pb2_grpc as cluster_pb2_grpc
|
|
5
|
+
import grpc
|
|
6
|
+
from grpc._channel import _InactiveRpcError
|
|
7
|
+
import multiprocessing
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _get_grpc_header(engine_ip=None, cluster=None):
|
|
11
|
+
metadata = []
|
|
12
|
+
if engine_ip:
|
|
13
|
+
metadata.append(('plannerip', engine_ip))
|
|
14
|
+
if cluster:
|
|
15
|
+
metadata.append(('cluster-uuid', cluster))
|
|
16
|
+
return metadata
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class _StatusLock:
|
|
20
|
+
"""
|
|
21
|
+
A thread-safe and process-safe lock manager designed for managing
|
|
22
|
+
concurrent access protection in multithreaded and multiprocessing environments.
|
|
23
|
+
|
|
24
|
+
This class encapsulates locking mechanisms using threading and multiprocessing
|
|
25
|
+
modules to ensure an operation's atomicity and handle shared resources safely.
|
|
26
|
+
|
|
27
|
+
Attributes:
|
|
28
|
+
_LOCK_TIMEOUT (int): The maximum timeout (in milliseconds) to try
|
|
29
|
+
acquiring a lock before raising an error.
|
|
30
|
+
_status_thread_lock (threading.Lock): A thread-level lock to synchronize
|
|
31
|
+
access among threads in the same process.
|
|
32
|
+
_status_multiprocessing_lock (multiprocessing.Semaphore): A
|
|
33
|
+
process-level lock to synchronize access across different processes.
|
|
34
|
+
_is_active (bool): A boolean flag indicating whether the lock
|
|
35
|
+
is currently active (True) or not (False).
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
_LOCK_TIMEOUT = 500
|
|
39
|
+
|
|
40
|
+
def __init__(self):
|
|
41
|
+
"""
|
|
42
|
+
Initializes the _StatusLock instance with its respective
|
|
43
|
+
thread and multiprocessing locks and sets the active flag to False.
|
|
44
|
+
"""
|
|
45
|
+
self._status_thread_lock = threading.Lock()
|
|
46
|
+
self._status_multiprocessing_lock = multiprocessing.Semaphore()
|
|
47
|
+
self._is_active = False
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def is_active(self) -> bool:
|
|
51
|
+
"""
|
|
52
|
+
Checks if the lock is currently active.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
bool: True if the lock is active, False otherwise.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
return self._is_active
|
|
59
|
+
|
|
60
|
+
def set_active(self):
|
|
61
|
+
"""
|
|
62
|
+
Activates the lock by setting the `_is_active` flag to True.
|
|
63
|
+
This can be used for manually marking the state of the lock
|
|
64
|
+
as active during synchronization operations.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
self._is_active = True
|
|
68
|
+
|
|
69
|
+
def __enter__(self):
|
|
70
|
+
"""
|
|
71
|
+
Enters a context-managed locking block.
|
|
72
|
+
|
|
73
|
+
Acquires both the thread-level lock and process-level semaphore
|
|
74
|
+
to ensure the current operation can safely access shared resources.
|
|
75
|
+
|
|
76
|
+
Raises:
|
|
77
|
+
TimeoutError: If the lock cannot be acquired within the timeout period.
|
|
78
|
+
|
|
79
|
+
Returns:
|
|
80
|
+
_StatusLock: The current instance of the lock, used for context management.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
self._status_thread_lock.acquire(timeout=self._LOCK_TIMEOUT)
|
|
84
|
+
self._status_multiprocessing_lock.acquire(timeout=self._LOCK_TIMEOUT)
|
|
85
|
+
return self
|
|
86
|
+
|
|
87
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
88
|
+
"""
|
|
89
|
+
Exits a context-managed locking block.
|
|
90
|
+
|
|
91
|
+
Releases both the thread-level lock and process-level semaphore
|
|
92
|
+
to allow other operations to acquire the lock. Handles cleanup
|
|
93
|
+
regardless of whether an exception occurred.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
exc_type (Type[BaseException]): The type of exception raised (if any).
|
|
97
|
+
exc_val (BaseException): The exception instance raised (if any).
|
|
98
|
+
exc_tb (Traceback): The traceback object of the exception (if any).
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
self._status_thread_lock.release()
|
|
102
|
+
self._status_multiprocessing_lock.release()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
status_lock = _StatusLock()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class ClusterManager:
|
|
109
|
+
"""
|
|
110
|
+
A manager for handling operations and connections with a remote cluster infrastructure.
|
|
111
|
+
Provides methods to manage cluster states, such as resuming or suspending its activity,
|
|
112
|
+
by interacting with a gRPC-based remote service.
|
|
113
|
+
|
|
114
|
+
Attributes:
|
|
115
|
+
_host (str): The hostname or IP address of the cluster service.
|
|
116
|
+
_port (int): The port number used to connect to the cluster service.
|
|
117
|
+
_user (str): The username for authentication with the cluster service.
|
|
118
|
+
_password (str): The password for authentication with the cluster service.
|
|
119
|
+
_timeout (float): The timeout threshold (as an epoch timestamp) for operations,
|
|
120
|
+
defaulting to 3 minutes from initialization.
|
|
121
|
+
_secure_channel (bool): Indicates whether a secure gRPC channel
|
|
122
|
+
(SSL/TLS) should be used for communication; defaults to False.
|
|
123
|
+
cluster_uuid (str): The unique identifier for the target cluster.
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 3, cluster_uuid=None):
|
|
127
|
+
"""
|
|
128
|
+
Initializes a new instance of the ClusterManager class.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
host (str): The hostname or IP address of the cluster service.
|
|
132
|
+
port (int): The port number for accessing the cluster service.
|
|
133
|
+
user (str): The username used for connecting to the cluster service.
|
|
134
|
+
password (str): The password used for connecting to the cluster service.
|
|
135
|
+
secure_channel (bool, optional): Whether to use a secure
|
|
136
|
+
gRPC channel for communication; defaults to False.
|
|
137
|
+
timeout (int, optional): The timeout duration (in seconds) for operations;
|
|
138
|
+
defaults to 3 minutes.
|
|
139
|
+
cluster_uuid (str, optional): The unique identifier for the target cluster;
|
|
140
|
+
defaults to None.
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
self._host = host
|
|
144
|
+
self._port = port
|
|
145
|
+
self._user = user
|
|
146
|
+
self._password = password
|
|
147
|
+
self._timeout = time.time() + timeout
|
|
148
|
+
self._secure_channel = secure_channel
|
|
149
|
+
self.cluster_uuid = cluster_uuid
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def _get_connection(self):
|
|
153
|
+
"""
|
|
154
|
+
Dynamically establishes a gRPC connection to the cluster service.
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
cluster_pb2_grpc.ClusterServiceStub: A gRPC client stub
|
|
158
|
+
for interacting with the cluster service methods.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
if self._secure_channel:
|
|
162
|
+
self._channel = grpc.secure_channel(
|
|
163
|
+
target='{}:{}'.format(self._host, self._port),
|
|
164
|
+
credentials=grpc.ssl_channel_credentials()
|
|
165
|
+
)
|
|
166
|
+
else:
|
|
167
|
+
self._channel = grpc.insecure_channel(
|
|
168
|
+
target='{}:{}'.format(self._host, self._port)
|
|
169
|
+
)
|
|
170
|
+
return cluster_pb2_grpc.ClusterServiceStub(self._channel)
|
|
171
|
+
|
|
172
|
+
def resume(self) -> bool:
|
|
173
|
+
"""
|
|
174
|
+
Resumes the cluster if it is currently suspended or not in the 'active' state.
|
|
175
|
+
|
|
176
|
+
This method interacts with the remote cluster service to verify its current
|
|
177
|
+
status. If suspended, it sends a resume request and monitors
|
|
178
|
+
the cluster's state until it becomes active or fails.
|
|
179
|
+
|
|
180
|
+
The operation uses a locking mechanism (`status_lock`) to ensure thread-safe
|
|
181
|
+
and process-safe state transitions.
|
|
182
|
+
|
|
183
|
+
Returns:
|
|
184
|
+
bool: True if the cluster resumes successfully and becomes active;
|
|
185
|
+
False if the cluster cannot be resumed, remains suspended, or fails.
|
|
186
|
+
|
|
187
|
+
Raises:
|
|
188
|
+
_InactiveRpcError: An exception raised if there is a communication error
|
|
189
|
+
while interacting with the remote cluster service.
|
|
190
|
+
|
|
191
|
+
Notes:
|
|
192
|
+
- If the cluster is already active, this method completes successfully
|
|
193
|
+
without further actions.
|
|
194
|
+
- If the cluster is in a 'resuming' state, this method waits for the
|
|
195
|
+
cluster to transition to 'active' or any terminal state (e.g., 'failed').
|
|
196
|
+
- The operation is subject to the `_timeout` threshold;
|
|
197
|
+
if the timeout expires, the method returns False.
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
with status_lock as lock:
|
|
201
|
+
if lock.is_active:
|
|
202
|
+
return True
|
|
203
|
+
|
|
204
|
+
# Retrieve the current cluster status
|
|
205
|
+
status_payload = cluster_pb2.ClusterStatusRequest(
|
|
206
|
+
user=self._user,
|
|
207
|
+
password=self._password
|
|
208
|
+
)
|
|
209
|
+
current_status = self._get_connection.status(
|
|
210
|
+
status_payload,
|
|
211
|
+
metadata=_get_grpc_header(cluster=self.cluster_uuid)
|
|
212
|
+
)
|
|
213
|
+
if current_status.status == 'suspended':
|
|
214
|
+
# Send the resume request
|
|
215
|
+
payload = cluster_pb2.ResumeRequest(
|
|
216
|
+
user=self._user,
|
|
217
|
+
password=self._password
|
|
218
|
+
)
|
|
219
|
+
response = self._get_connection.resume(
|
|
220
|
+
payload,
|
|
221
|
+
metadata=_get_grpc_header(cluster=self.cluster_uuid)
|
|
222
|
+
)
|
|
223
|
+
elif current_status.status == 'active':
|
|
224
|
+
return True
|
|
225
|
+
elif current_status.status != 'resuming':
|
|
226
|
+
"""
|
|
227
|
+
If cluster cannot be resumed due to its current state,
|
|
228
|
+
or already in a process of resuming, terminate the operation.
|
|
229
|
+
"""
|
|
230
|
+
return False
|
|
231
|
+
|
|
232
|
+
# Wait for the cluster to become active
|
|
233
|
+
while True:
|
|
234
|
+
try:
|
|
235
|
+
status_payload = cluster_pb2.ClusterStatusRequest(
|
|
236
|
+
user=self._user,
|
|
237
|
+
password=self._password
|
|
238
|
+
)
|
|
239
|
+
response = self._get_connection.status(
|
|
240
|
+
status_payload,
|
|
241
|
+
metadata=_get_grpc_header(cluster=self.cluster_uuid)
|
|
242
|
+
)
|
|
243
|
+
if response.status == 'active':
|
|
244
|
+
lock.set_active()
|
|
245
|
+
return True
|
|
246
|
+
if response.status in ['suspended', 'failed']:
|
|
247
|
+
return False
|
|
248
|
+
if time.time() > self._timeout:
|
|
249
|
+
return False
|
|
250
|
+
except _InactiveRpcError as e:
|
|
251
|
+
pass
|
|
252
|
+
time.sleep(5)
|
|
253
|
+
|
|
254
|
+
def suspend(self):
|
|
255
|
+
"""
|
|
256
|
+
Suspends the cluster operations (not implemented).
|
|
257
|
+
|
|
258
|
+
Placeholder method to be implemented for suspending a cluster,
|
|
259
|
+
typically interacting with the remote cluster service API.
|
|
260
|
+
"""
|
|
261
|
+
pass
|
|
@@ -53,14 +53,35 @@ def _parse_timestamp(value):
|
|
|
53
53
|
value = datetime.datetime.strptime(value, format)
|
|
54
54
|
else:
|
|
55
55
|
raise Exception(
|
|
56
|
-
'Cannot convert "{}" into a datetime'.format(value)
|
|
56
|
+
'Cannot convert "{}" into a datetime'.format(value)
|
|
57
|
+
)
|
|
57
58
|
else:
|
|
58
59
|
value = None
|
|
59
60
|
return value
|
|
60
61
|
|
|
61
62
|
|
|
62
|
-
TYPES_CONVERTER = {
|
|
63
|
-
|
|
63
|
+
TYPES_CONVERTER = {
|
|
64
|
+
"DECIMAL_TYPE": Decimal,
|
|
65
|
+
"TIMESTAMP_TYPE": _parse_timestamp
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def re_auth(func):
|
|
70
|
+
def wrapper(self, *args, **kwargs):
|
|
71
|
+
try:
|
|
72
|
+
return func(self, *args, **kwargs)
|
|
73
|
+
except _InactiveRpcError as e:
|
|
74
|
+
print(f'RE_AUTH: Function Name: {func}')
|
|
75
|
+
print(f'RE_AUTH: Error Found {e}')
|
|
76
|
+
if e.code() == grpc.StatusCode.INTERNAL and 'Access denied' in e.details():
|
|
77
|
+
print('RE_AUTH: Initialising re-authentication.')
|
|
78
|
+
self.connection.get_re_authenticate_session_id()
|
|
79
|
+
print(f'RE_AUTH: Re-auth successful.')
|
|
80
|
+
return func(self, *args, **kwargs)
|
|
81
|
+
else:
|
|
82
|
+
raise e
|
|
83
|
+
|
|
84
|
+
return wrapper
|
|
64
85
|
|
|
65
86
|
|
|
66
87
|
class HiveParamEscaper(ParamEscaper):
|
|
@@ -148,8 +169,6 @@ class Connection(object):
|
|
|
148
169
|
- max_receive_message_length: This parameter sets the maximum allowed size (in bytes) for incoming messages on the gRPC server.
|
|
149
170
|
- max_send_message_length: Similar to max_receive_message_length, this parameter sets the maximum allowed size (in bytes) for outgoing messages from the gRPC client
|
|
150
171
|
- grpc_prepare_timeout: Timeout for prepare statement API call (default to 10 minutes).
|
|
151
|
-
#keep-alive parameters
|
|
152
|
-
- keepalive_time_ms: This parameter defines the time, in milliseconds, Default to 30 seconds
|
|
153
172
|
"""
|
|
154
173
|
if not username or not password:
|
|
155
174
|
raise ValueError("username or password cannot be empty.")
|
|
@@ -170,18 +189,16 @@ class Connection(object):
|
|
|
170
189
|
self._auto_resume = auto_resume
|
|
171
190
|
|
|
172
191
|
self._keepalive_timeout_ms = 900000
|
|
173
|
-
self._keepalive_time_ms = 30000
|
|
174
192
|
self._max_receive_message_length = -1
|
|
175
193
|
self._max_send_message_length = 300 * 1024 * 1024 # mb
|
|
176
194
|
self.grpc_prepare_timeout = 10 * 60 # 10 minutes
|
|
177
195
|
|
|
178
|
-
if
|
|
196
|
+
if isinstance(grpc_options, dict):
|
|
179
197
|
self._keepalive_timeout_ms = grpc_options.get('keepalive_timeout_ms') or self._keepalive_timeout_ms
|
|
180
198
|
self._max_receive_message_length = grpc_options.get(
|
|
181
199
|
'max_receive_message_length') or self._max_receive_message_length
|
|
182
200
|
self._max_send_message_length = grpc_options.get('max_send_message_length') or self._max_send_message_length
|
|
183
201
|
self.grpc_prepare_timeout = grpc_options.get('grpc_prepare_timeout') or self.grpc_prepare_timeout
|
|
184
|
-
self._keepalive_time_ms = grpc_options.get('keepalive_time_ms') or self._keepalive_time_ms
|
|
185
202
|
self._create_client()
|
|
186
203
|
|
|
187
204
|
def _create_client(self):
|
|
@@ -191,12 +208,7 @@ class Connection(object):
|
|
|
191
208
|
options=[
|
|
192
209
|
("grpc.keepalive_timeout_ms", self._keepalive_timeout_ms),
|
|
193
210
|
('grpc.max_send_message_length', self._max_send_message_length),
|
|
194
|
-
('grpc.max_receive_message_length', self._max_receive_message_length)
|
|
195
|
-
('grpc.keepalive_time_ms', self._keepalive_time_ms),
|
|
196
|
-
('grpc.keepalive_permit_without_calls', 1), # Allow keep-alives with no active RPCs
|
|
197
|
-
('grpc.http2.max_pings_without_data', 0), # Unlimited pings without data
|
|
198
|
-
('grpc.http2.min_time_between_pings_ms', 15000), # Minimum 15 seconds between pings
|
|
199
|
-
('grpc.http2.min_ping_interval_without_data_ms', 15000)# Minimum 15 seconds between pings without data
|
|
211
|
+
('grpc.max_receive_message_length', self._max_receive_message_length)
|
|
200
212
|
],
|
|
201
213
|
credentials=grpc.ssl_channel_credentials()
|
|
202
214
|
)
|
|
@@ -206,16 +218,15 @@ class Connection(object):
|
|
|
206
218
|
options=[
|
|
207
219
|
("grpc.keepalive_timeout_ms", self._keepalive_timeout_ms),
|
|
208
220
|
('grpc.max_send_message_length', self._max_send_message_length),
|
|
209
|
-
('grpc.max_receive_message_length', self._max_receive_message_length)
|
|
210
|
-
('grpc.keepalive_time_ms', self._keepalive_time_ms),
|
|
211
|
-
('grpc.keepalive_permit_without_calls', 1), # Allow keep-alives with no active RPCs
|
|
212
|
-
('grpc.http2.max_pings_without_data', 0), # Unlimited pings without data
|
|
213
|
-
('grpc.http2.min_time_between_pings_ms', 15000), # Minimum 15 seconds between pings
|
|
214
|
-
('grpc.http2.min_ping_interval_without_data_ms', 15000)# Minimum 15 seconds between pings without data
|
|
221
|
+
('grpc.max_receive_message_length', self._max_receive_message_length)
|
|
215
222
|
]
|
|
216
223
|
)
|
|
217
224
|
self._client = e6x_engine_pb2_grpc.QueryEngineServiceStub(self._channel)
|
|
218
225
|
|
|
226
|
+
def get_re_authenticate_session_id(self):
|
|
227
|
+
self._session_id = None
|
|
228
|
+
return self.get_session_id
|
|
229
|
+
|
|
219
230
|
@property
|
|
220
231
|
def get_session_id(self):
|
|
221
232
|
"""
|
|
@@ -236,13 +247,14 @@ class Connection(object):
|
|
|
236
247
|
raise ValueError("Invalid credentials.")
|
|
237
248
|
except _InactiveRpcError as e:
|
|
238
249
|
if self._auto_resume:
|
|
239
|
-
if e.code() == grpc.StatusCode.
|
|
250
|
+
if e.code() == grpc.StatusCode.UNAVAILABLE and 'status: 503' in e.details():
|
|
240
251
|
status = ClusterManager(
|
|
241
252
|
host=self._host,
|
|
242
253
|
port=self._port,
|
|
243
254
|
user=self.__username,
|
|
244
255
|
password=self.__password,
|
|
245
|
-
secure_channel=self._secure_channel
|
|
256
|
+
secure_channel=self._secure_channel,
|
|
257
|
+
cluster_uuid=self.cluster_uuid
|
|
246
258
|
).resume()
|
|
247
259
|
if status:
|
|
248
260
|
authenticate_request = e6x_engine_pb2.AuthenticateRequest(
|
|
@@ -510,6 +522,7 @@ class Cursor(DBAPICursor):
|
|
|
510
522
|
)
|
|
511
523
|
return self.connection.client.status(status_request, metadata=self.metadata)
|
|
512
524
|
|
|
525
|
+
@re_auth
|
|
513
526
|
def execute(self, operation, parameters=None, **kwargs):
|
|
514
527
|
"""Prepare and execute a database operation (query or command).
|
|
515
528
|
Return values are not defined.
|
|
@@ -565,6 +578,7 @@ class Cursor(DBAPICursor):
|
|
|
565
578
|
|
|
566
579
|
self._query_id = prepare_statement_response.queryId
|
|
567
580
|
self._engine_ip = prepare_statement_response.engineIP
|
|
581
|
+
|
|
568
582
|
execute_statement_request = e6x_engine_pb2.ExecuteStatementV2Request(
|
|
569
583
|
engineIP=self._engine_ip,
|
|
570
584
|
sessionId=self.connection.get_session_id,
|
|
@@ -649,7 +663,6 @@ class Cursor(DBAPICursor):
|
|
|
649
663
|
return self._fetch_all()
|
|
650
664
|
|
|
651
665
|
def fetchmany(self, size: int = None):
|
|
652
|
-
# _logger.info("fetching all from overriden method")
|
|
653
666
|
if size is None:
|
|
654
667
|
size = self.arraysize
|
|
655
668
|
if self._data is None:
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.2
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
|
-
Author:
|
|
6
|
+
Author: e6data, Inc.
|
|
7
7
|
Author-email: info@e6data.com
|
|
8
8
|
License: Apache 2.0
|
|
9
9
|
Classifier: Operating System :: POSIX :: Linux
|
|
@@ -13,21 +13,32 @@ Classifier: Programming Language :: Python :: 3.6
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.7
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.8
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
19
|
Description-Content-Type: text/markdown
|
|
17
20
|
License-File: LICENSE
|
|
18
21
|
Requires-Dist: sqlalchemy>=1.0.0
|
|
19
|
-
Requires-Dist: future
|
|
20
|
-
Requires-Dist: python-dateutil
|
|
21
|
-
Requires-Dist: pycryptodome
|
|
22
|
-
Requires-Dist: pytz
|
|
23
|
-
Requires-Dist: thrift
|
|
24
|
-
Requires-Dist: grpcio
|
|
22
|
+
Requires-Dist: future==1.0.0
|
|
23
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
24
|
+
Requires-Dist: pycryptodome==3.19.1
|
|
25
|
+
Requires-Dist: pytz==2024.1
|
|
26
|
+
Requires-Dist: thrift==0.20.0
|
|
27
|
+
Requires-Dist: grpcio==1.65.1
|
|
25
28
|
Requires-Dist: grpcio-tools
|
|
26
|
-
|
|
29
|
+
Dynamic: author
|
|
30
|
+
Dynamic: author-email
|
|
31
|
+
Dynamic: classifier
|
|
32
|
+
Dynamic: description
|
|
33
|
+
Dynamic: description-content-type
|
|
34
|
+
Dynamic: home-page
|
|
35
|
+
Dynamic: license
|
|
36
|
+
Dynamic: requires-dist
|
|
37
|
+
Dynamic: summary
|
|
27
38
|
|
|
28
39
|
# e6data Python Connector
|
|
29
40
|
|
|
30
|
-

|
|
31
42
|
|
|
32
43
|
## Introduction
|
|
33
44
|
|
|
@@ -12,37 +12,45 @@
|
|
|
12
12
|
|
|
13
13
|
import setuptools
|
|
14
14
|
|
|
15
|
-
VERSION =
|
|
15
|
+
VERSION = (2, 2, 2)
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
def get_long_desc():
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
"""
|
|
20
|
+
Reads the `README.md` file to provide a detailed description of the package.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
str: The content of the README.md file as a string.
|
|
24
|
+
"""
|
|
25
|
+
try:
|
|
26
|
+
with open("README.md", "r", encoding="utf-8") as fh: # Explicitly specifying encoding.
|
|
27
|
+
return fh.read()
|
|
28
|
+
except FileNotFoundError:
|
|
29
|
+
# Fallback if README.md is missing.
|
|
30
|
+
return "Client for the e6data distributed SQL Engine."
|
|
22
31
|
|
|
23
32
|
|
|
24
33
|
setuptools.setup(
|
|
25
34
|
name="e6data-python-connector",
|
|
26
|
-
version=
|
|
27
|
-
author="
|
|
35
|
+
version=".".join(map(str, VERSION)), # More Pythonic way to convert tuple to string.
|
|
36
|
+
author="e6data, Inc.",
|
|
28
37
|
author_email="info@e6data.com",
|
|
29
38
|
description="Client for the e6data distributed SQL Engine.",
|
|
30
39
|
long_description=get_long_desc(),
|
|
31
40
|
long_description_content_type="text/markdown",
|
|
32
41
|
url='https://github.com/e6x-labs/e6data-python-connector',
|
|
33
|
-
packages=setuptools.find_packages(),
|
|
42
|
+
packages=setuptools.find_packages(), # Automatically detects all packages in the source tree.
|
|
34
43
|
license="Apache 2.0",
|
|
35
|
-
include_package_data=True,
|
|
44
|
+
include_package_data=True, # Includes non-Python files specified in MANIFEST.in.
|
|
36
45
|
install_requires=[
|
|
37
46
|
'sqlalchemy>=1.0.0',
|
|
38
|
-
'future',
|
|
39
|
-
'python-dateutil',
|
|
40
|
-
'pycryptodome',
|
|
41
|
-
'pytz',
|
|
42
|
-
'thrift',
|
|
43
|
-
'grpcio',
|
|
47
|
+
'future==1.0.0',
|
|
48
|
+
'python-dateutil==2.9.0.post0',
|
|
49
|
+
'pycryptodome==3.19.1',
|
|
50
|
+
'pytz==2024.1',
|
|
51
|
+
'thrift==0.20.0',
|
|
52
|
+
'grpcio==1.65.1',
|
|
44
53
|
'grpcio-tools',
|
|
45
|
-
'sqlalchemy'
|
|
46
54
|
],
|
|
47
55
|
classifiers=[
|
|
48
56
|
"Operating System :: POSIX :: Linux",
|
|
@@ -51,7 +59,10 @@ setuptools.setup(
|
|
|
51
59
|
'Programming Language :: Python :: 3.6',
|
|
52
60
|
'Programming Language :: Python :: 3.7',
|
|
53
61
|
'Programming Language :: Python :: 3.8',
|
|
54
|
-
'Programming Language :: Python :: 3.9'
|
|
62
|
+
'Programming Language :: Python :: 3.9',
|
|
63
|
+
"Programming Language :: Python :: 3.10",
|
|
64
|
+
"Programming Language :: Python :: 3.11",
|
|
65
|
+
"Programming Language :: Python :: 3.12",
|
|
55
66
|
],
|
|
56
67
|
entry_points={
|
|
57
68
|
'sqlalchemy.dialects': [
|
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
import threading
|
|
2
|
-
import time
|
|
3
|
-
import e6data_python_connector.cluster_server.cluster_pb2 as cluster_pb2
|
|
4
|
-
import e6data_python_connector.cluster_server.cluster_pb2_grpc as cluster_pb2_grpc
|
|
5
|
-
import grpc
|
|
6
|
-
from grpc._channel import _InactiveRpcError
|
|
7
|
-
import multiprocessing
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class _StatusLock:
|
|
11
|
-
_LOCK_TIMEOUT = 500
|
|
12
|
-
|
|
13
|
-
def __init__(self):
|
|
14
|
-
self._status_thread_lock = threading.Lock()
|
|
15
|
-
self._status_multiprocessing_lock = multiprocessing.Semaphore()
|
|
16
|
-
self._is_active = False
|
|
17
|
-
|
|
18
|
-
@property
|
|
19
|
-
def is_active(self) -> bool:
|
|
20
|
-
return self._is_active
|
|
21
|
-
|
|
22
|
-
def set_active(self):
|
|
23
|
-
self._is_active = True
|
|
24
|
-
|
|
25
|
-
def __enter__(self):
|
|
26
|
-
self._status_thread_lock.acquire(timeout=self._LOCK_TIMEOUT)
|
|
27
|
-
self._status_multiprocessing_lock.acquire(timeout=self._LOCK_TIMEOUT)
|
|
28
|
-
return self
|
|
29
|
-
|
|
30
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
31
|
-
self._status_thread_lock.release()
|
|
32
|
-
self._status_multiprocessing_lock.release()
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
status_lock = _StatusLock()
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
class ClusterManager:
|
|
39
|
-
def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 3):
|
|
40
|
-
self._host = host
|
|
41
|
-
self._port = port
|
|
42
|
-
self._user = user
|
|
43
|
-
self._password = password
|
|
44
|
-
self._timeout = time.time() + timeout
|
|
45
|
-
self._secure_channel = secure_channel
|
|
46
|
-
|
|
47
|
-
@property
|
|
48
|
-
def _get_connection(self):
|
|
49
|
-
if self._secure_channel:
|
|
50
|
-
self._channel = grpc.secure_channel(
|
|
51
|
-
target='{}:{}'.format(self._host, self._port),
|
|
52
|
-
credentials=grpc.ssl_channel_credentials()
|
|
53
|
-
)
|
|
54
|
-
else:
|
|
55
|
-
self._channel = grpc.insecure_channel(
|
|
56
|
-
target='{}:{}'.format(self._host, self._port)
|
|
57
|
-
)
|
|
58
|
-
return cluster_pb2_grpc.ClusterServiceStub(self._channel)
|
|
59
|
-
|
|
60
|
-
def resume(self) -> bool:
|
|
61
|
-
"""
|
|
62
|
-
Return True if cluster came to active state else False.
|
|
63
|
-
"""
|
|
64
|
-
with status_lock as lock:
|
|
65
|
-
if lock.is_active:
|
|
66
|
-
return True
|
|
67
|
-
status_payload = cluster_pb2.ClusterStatusRequest(
|
|
68
|
-
user=self._user,
|
|
69
|
-
password=self._password
|
|
70
|
-
)
|
|
71
|
-
current_status = self._get_connection.status(status_payload)
|
|
72
|
-
if current_status.status == 'suspended':
|
|
73
|
-
payload = cluster_pb2.ResumeRequest(
|
|
74
|
-
user=self._user,
|
|
75
|
-
password=self._password
|
|
76
|
-
)
|
|
77
|
-
response = self._get_connection.resume(payload)
|
|
78
|
-
elif current_status.status == 'active':
|
|
79
|
-
return True
|
|
80
|
-
elif current_status.status != 'resuming':
|
|
81
|
-
"""
|
|
82
|
-
If cluster is in resuming state already, start watching for the status.
|
|
83
|
-
Cluster is in different state, cannot resume.
|
|
84
|
-
"""
|
|
85
|
-
return False
|
|
86
|
-
while True:
|
|
87
|
-
try:
|
|
88
|
-
status_payload = cluster_pb2.ClusterStatusRequest(
|
|
89
|
-
user=self._user,
|
|
90
|
-
password=self._password
|
|
91
|
-
)
|
|
92
|
-
response = self._get_connection.status(status_payload)
|
|
93
|
-
if response.status == 'active':
|
|
94
|
-
lock.set_active()
|
|
95
|
-
return True
|
|
96
|
-
if response.status in ['suspended', 'failed']:
|
|
97
|
-
return False
|
|
98
|
-
if time.time() > self._timeout:
|
|
99
|
-
return False
|
|
100
|
-
except _InactiveRpcError as e:
|
|
101
|
-
pass
|
|
102
|
-
time.sleep(5)
|
|
103
|
-
|
|
104
|
-
def suspend(self):
|
|
105
|
-
pass
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/common.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{e6data_python_connector-2.2.1rc5 → e6data_python_connector-2.2.2}/e6data_python_connector/typeId.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|