e6data-python-connector 2.1.21__tar.gz → 2.1.21.dev0__tar.gz

{e6data_python_connector-2.1.21/e6data_python_connector.egg-info → e6data_python_connector-2.1.21.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: e6data-python-connector
-Version: 2.1.21
+Version: 2.1.21.dev0
 Summary: Client for the e6data distributed SQL Engine.
 Home-page: https://github.com/e6x-labs/e6data-python-connector
 Author: Uniphi, Inc.

e6data_python_connector-2.1.21.dev0/e6data_python_connector/cluster_manager.py

@@ -0,0 +1,117 @@
+import threading
+import time
+import e6data_python_connector.cluster_server.cluster_pb2 as cluster_pb2
+import e6data_python_connector.cluster_server.cluster_pb2_grpc as cluster_pb2_grpc
+import grpc
+from grpc._channel import _InactiveRpcError
+import multiprocessing
+
+
+class _StatusLock:
+    _LOCK_TIMEOUT = 500
+
+    def __init__(self):
+        self._status_thread_lock = threading.Lock()
+        self._status_multiprocessing_lock = multiprocessing.Semaphore()
+        self._is_active = False
+
+    @property
+    def is_active(self) -> bool:
+        return self._is_active
+
+    def set_active(self):
+        self._is_active = True
+
+    def __enter__(self):
+        self._status_thread_lock.acquire(timeout=self._LOCK_TIMEOUT)
+        self._status_multiprocessing_lock.acquire(timeout=self._LOCK_TIMEOUT)
+        print('Locked threads and processes.')
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._status_thread_lock.release()
+        self._status_multiprocessing_lock.release()
+        print('Released threads and processes.')
+
+
+status_lock = _StatusLock()
+
+
+class ClusterManager:
+    def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 3):
+        self._host = host
+        self._port = port
+        self._user = user
+        self._password = password
+        self._timeout = time.time() + timeout
+        self._secure_channel = secure_channel
+
+    @property
+    def _get_connection(self):
+        if self._secure_channel:
+            self._channel = grpc.secure_channel(
+                target='{}:{}'.format(self._host, self._port),
+                credentials=grpc.ssl_channel_credentials()
+            )
+        else:
+            self._channel = grpc.insecure_channel(
+                target='{}:{}'.format(self._host, self._port)
+            )
+        return cluster_pb2_grpc.ClusterServiceStub(self._channel)
+
+    def resume(self) -> bool:
+        """
+        Return True if cluster came to active state else False.
+        """
+        with status_lock as lock:
+            if lock.is_active:
+                print('Cluster is already active')
+                return True
+            status_payload = cluster_pb2.ClusterStatusRequest(
+                user=self._user,
+                password=self._password
+            )
+            current_status = self._get_connection.status(status_payload)
+            print('Current status', current_status)
+            if current_status.status == 'suspended':
+                payload = cluster_pb2.ResumeRequest(
+                    user=self._user,
+                    password=self._password
+                )
+                response = self._get_connection.resume(payload)
+                print('resuming response', response.status)
+            elif current_status.status == 'active':
+                print('Cluster is already active.')
+                return True
+            elif current_status.status != 'resuming':
+                """
+                If cluster is in resuming state already, start watching for the status.
+                Cluster is in different state, cannot resume.
+                """
+                print('Cluster is in different state, cannot resume.')
+                return False
+            while True:
+                try:
+                    status_payload = cluster_pb2.ClusterStatusRequest(
+                        user=self._user,
+                        password=self._password
+                    )
+                    response = self._get_connection.status(status_payload)
+                    print(response)
+                    if response.status == 'active':
+                        lock.set_active()
+                        print('Breaking because of status:', response.status)
+                        return True
+                    if response.status in ['suspended', 'failed']:
+                        print('Breaking because of status:', response.status)
+                        return False
+                    if time.time() > self._timeout:
+                        print('Breaking because of timeout')
+                        return False
+                except _InactiveRpcError as e:
+                    print('On resume error', e)
+                    print('Retrying')
+                time.sleep(5)
+
+    def suspend(self):
+        pass

e6data_python_connector-2.1.21.dev0/e6data_python_connector/cluster_server/cluster_pb2.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: cluster.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rcluster.proto\"/\n\rResumeRequest\x12\x0c\n\x04user\x18\x01 \x01(\t\x12\x10\n\x08password\x18\x02 \x01(\t\" \n\x0eResumeResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"0\n\x0eSuspendRequest\x12\x0c\n\x04user\x18\x01 \x01(\t\x12\x10\n\x08password\x18\x02 \x01(\t\"!\n\x0fSuspendResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\"6\n\x14\x43lusterStatusRequest\x12\x0c\n\x04user\x18\x01 \x01(\t\x12\x10\n\x08password\x18\x02 \x01(\t\"\'\n\x15\x43lusterStatusResponse\x12\x0e\n\x06status\x18\x01 \x01(\t2\xa2\x01\n\x0e\x43lusterService\x12)\n\x06resume\x12\x0e.ResumeRequest\x1a\x0f.ResumeResponse\x12,\n\x07suspend\x12\x0f.SuspendRequest\x1a\x10.SuspendResponse\x12\x37\n\x06status\x12\x15.ClusterStatusRequest\x1a\x16.ClusterStatusResponseB\x02P\x01\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'cluster_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  DESCRIPTOR._options = None
+  DESCRIPTOR._serialized_options = b'P\001'
+  _globals['_RESUMEREQUEST']._serialized_start=17
+  _globals['_RESUMEREQUEST']._serialized_end=64
+  _globals['_RESUMERESPONSE']._serialized_start=66
+  _globals['_RESUMERESPONSE']._serialized_end=98
+  _globals['_SUSPENDREQUEST']._serialized_start=100
+  _globals['_SUSPENDREQUEST']._serialized_end=148
+  _globals['_SUSPENDRESPONSE']._serialized_start=150
+  _globals['_SUSPENDRESPONSE']._serialized_end=183
+  _globals['_CLUSTERSTATUSREQUEST']._serialized_start=185
+  _globals['_CLUSTERSTATUSREQUEST']._serialized_end=239
+  _globals['_CLUSTERSTATUSRESPONSE']._serialized_start=241
+  _globals['_CLUSTERSTATUSRESPONSE']._serialized_end=280
+  _globals['_CLUSTERSERVICE']._serialized_start=283
+  _globals['_CLUSTERSERVICE']._serialized_end=445
+# @@protoc_insertion_point(module_scope)

e6data_python_connector-2.1.21.dev0/e6data_python_connector/cluster_server/cluster_pb2_grpc.py

@@ -0,0 +1,132 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+import e6data_python_connector.cluster_server.cluster_pb2 as cluster__pb2
+
+
+class ClusterServiceStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.resume = channel.unary_unary(
+                '/ClusterService/resume',
+                request_serializer=cluster__pb2.ResumeRequest.SerializeToString,
+                response_deserializer=cluster__pb2.ResumeResponse.FromString,
+                )
+        self.suspend = channel.unary_unary(
+                '/ClusterService/suspend',
+                request_serializer=cluster__pb2.SuspendRequest.SerializeToString,
+                response_deserializer=cluster__pb2.SuspendResponse.FromString,
+                )
+        self.status = channel.unary_unary(
+                '/ClusterService/status',
+                request_serializer=cluster__pb2.ClusterStatusRequest.SerializeToString,
+                response_deserializer=cluster__pb2.ClusterStatusResponse.FromString,
+                )
+
+
+class ClusterServiceServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def resume(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def suspend(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def status(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_ClusterServiceServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'resume': grpc.unary_unary_rpc_method_handler(
+                    servicer.resume,
+                    request_deserializer=cluster__pb2.ResumeRequest.FromString,
+                    response_serializer=cluster__pb2.ResumeResponse.SerializeToString,
+            ),
+            'suspend': grpc.unary_unary_rpc_method_handler(
+                    servicer.suspend,
+                    request_deserializer=cluster__pb2.SuspendRequest.FromString,
+                    response_serializer=cluster__pb2.SuspendResponse.SerializeToString,
+            ),
+            'status': grpc.unary_unary_rpc_method_handler(
+                    servicer.status,
+                    request_deserializer=cluster__pb2.ClusterStatusRequest.FromString,
+                    response_serializer=cluster__pb2.ClusterStatusResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'ClusterService', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class ClusterService(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def resume(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/ClusterService/resume',
+            cluster__pb2.ResumeRequest.SerializeToString,
+            cluster__pb2.ResumeResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def suspend(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/ClusterService/suspend',
+            cluster__pb2.SuspendRequest.SerializeToString,
+            cluster__pb2.SuspendResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def status(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/ClusterService/status',
+            cluster__pb2.ClusterStatusRequest.SerializeToString,
+            cluster__pb2.ClusterStatusResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

{e6data_python_connector-2.1.21 → e6data_python_connector-2.1.21.dev0}/e6data_python_connector/e6data_grpc.py

@@ -16,7 +16,9 @@ from io import BytesIO
 from ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
 
 import grpc
+from grpc._channel import _InactiveRpcError
 
+from e6data_python_connector.cluster_manager import ClusterManager
 from e6data_python_connector.common import DBAPITypeObject, ParamEscaper, DBAPICursor
 from e6data_python_connector.constants import *
 from e6data_python_connector.datainputstream import get_query_columns_info, read_rows_from_chunk
@@ -114,6 +116,7 @@ class Connection(object):
             database: str = None,
             cluster_uuid: str = None,
             secure: bool = False,
+            auto_resume: bool = False,
             grpc_options: dict = None,
     ):
         """
@@ -137,6 +140,8 @@ class Connection(object):
                 Cluster's uuid
             secure: bool, Optional
                 Flag to use a secure channel for data transfer
+            auto_resume: bool, Optional
+                Flag to enable auto resume of the cluster before the query execution
             grpc_options: dict, Optional
                 Specify gRPC configuration
                 - keepalive_timeout_ms: This parameter defines the time, in milliseconds, that a gRPC connection should remain idle before sending a keepalive ping to check if the connection is still alive.
@@ -160,6 +165,8 @@ class Connection(object):
 
         self.catalog_name = catalog
 
+        self._auto_resume = auto_resume
+
         self._keepalive_timeout_ms = 900000
         self._max_receive_message_length = -1
         self._max_send_message_length = 300 * 1024 * 1024  # mb
@@ -208,11 +215,38 @@ class Connection(object):
                 )
                 authenticate_response = self._client.authenticate(
                     authenticate_request,
-                    metadata=_get_grpc_header(cluster=self.cluster_uuid)
+                    metadata=_get_grpc_header(cluster=self.cluster_uuid),
+                    # timeout=3
                 )
                 self._session_id = authenticate_response.sessionId
                 if not self._session_id:
                     raise ValueError("Invalid credentials.")
+            except _InactiveRpcError as e:
+                if self._auto_resume:
+                    if e.code() == grpc.StatusCode.UNKNOWN and 'Stream removed' in e.details():
+                        status = ClusterManager(
+                            host=self._host,
+                            port=self._port,
+                            user=self.__username,
+                            password=self.__password,
+                            secure_channel=self._secure_channel
+                        ).resume()
+                        if status:
+                            authenticate_request = e6x_engine_pb2.AuthenticateRequest(
+                                user=self.__username,
+                                password=self.__password
+                            )
+                            authenticate_response = self._client.authenticate(
+                                authenticate_request,
+                                metadata=_get_grpc_header(cluster=self.cluster_uuid)
+                            )
+                            self._session_id = authenticate_response.sessionId
+                        else:
+                            raise e
+                    else:
+                        raise e
+                else:
+                    raise e
             except Exception as e:
                 self._channel.close()
                 raise e

{e6data_python_connector-2.1.21 → e6data_python_connector-2.1.21.dev0/e6data_python_connector.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: e6data-python-connector
-Version: 2.1.21
+Version: 2.1.21.dev0
 Summary: Client for the e6data distributed SQL Engine.
 Home-page: https://github.com/e6x-labs/e6data-python-connector
 Author: Uniphi, Inc.

{e6data_python_connector-2.1.21 → e6data_python_connector-2.1.21.dev0}/e6data_python_connector.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ pyproject.toml
 setup.cfg
 setup.py
 e6data_python_connector/__init__.py
+e6data_python_connector/cluster_manager.py
 e6data_python_connector/common.py
 e6data_python_connector/constants.py
 e6data_python_connector/datainputstream.py
@@ -19,6 +20,9 @@ e6data_python_connector.egg-info/dependency_links.txt
 e6data_python_connector.egg-info/entry_points.txt
 e6data_python_connector.egg-info/requires.txt
 e6data_python_connector.egg-info/top_level.txt
+e6data_python_connector/cluster_server/__init__.py
+e6data_python_connector/cluster_server/cluster_pb2.py
+e6data_python_connector/cluster_server/cluster_pb2_grpc.py
 e6data_python_connector/e6x_vector/__init__.py
 e6data_python_connector/e6x_vector/constants.py
 e6data_python_connector/e6x_vector/ttypes.py

{e6data_python_connector-2.1.21 → e6data_python_connector-2.1.21.dev0}/setup.py

@@ -12,7 +12,7 @@
 
 import setuptools
 
-VERSION = [2, 1, 21]
+VERSION = [2, 1, 21, 'dev']
 
 
 def get_long_desc():