e6data-python-connector 1.1.12__tar.gz → 2.0.0__tar.gz

{e6data-python-connector-1.1.12/e6data_python_connector.egg-info → e6data-python-connector-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: e6data-python-connector
-Version: 1.1.12
+Version: 2.0.0
 Summary: Client for the e6data distributed SQL Engine.
 Home-page: https://github.com/e6x-labs/e6data-python-connector
 Author: Uniphi, Inc.
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: sqlalchemy>=1.0.0
+Requires-Dist: future
 Requires-Dist: python-dateutil
 Requires-Dist: pycryptodome
 Requires-Dist: pytz
@@ -26,21 +27,23 @@ Requires-Dist: sqlalchemy
 
 # e6data Python Connector
 
-![version](https://img.shields.io/badge/version-1.1.12-blue.svg)
+![version](https://img.shields.io/badge/version-2.0.0-blue.svg)
 
 ## Introduction
 
 The e6data Connector for Python provides an interface for writing Python applications that can connect to e6data and perform operations.
 
-#### Amazon Linux Wheel dependency
-Make sure to install wheel before install e6data-python-connector.
+#### Amazon Linux dependency
+Make sure to install C compiler dependencies and wheel before install e6data-python-connector.
 ```shell
+yum install python3-devel gcc-c++ -y
+
 pip install wheel
 ```
 
 To install the Python package, use the command below:
 ```shell
-pip install e6data-python-connector
+pip install --no-cache-dir e6data-python-connector
 ```
 ### Prerequisites
 

{e6data-python-connector-1.1.12 → e6data-python-connector-2.0.0}/README.md

@@ -1,20 +1,22 @@
 # e6data Python Connector
 
-![version](https://img.shields.io/badge/version-1.1.12-blue.svg)
+![version](https://img.shields.io/badge/version-2.0.0-blue.svg)
 
 ## Introduction
 
 The e6data Connector for Python provides an interface for writing Python applications that can connect to e6data and perform operations.
 
-#### Amazon Linux Wheel dependency
-Make sure to install wheel before install e6data-python-connector.
+#### Amazon Linux dependency
+Make sure to install C compiler dependencies and wheel before install e6data-python-connector.
 ```shell
+yum install python3-devel gcc-c++ -y
+
 pip install wheel
 ```
 
 To install the Python package, use the command below:
 ```shell
-pip install e6data-python-connector
+pip install --no-cache-dir e6data-python-connector
 ```
 ### Prerequisites
 

{e6data-python-connector-1.1.12 → e6data-python-connector-2.0.0}/e6data_python_connector/datainputstream.py

@@ -1,8 +1,14 @@
-import struct
 import logging
-from e6xdb.date_time_utils import FORMATS, floor_div, floor_mod
+import struct
 from datetime import datetime, timedelta
+
+import pytz
+from thrift.protocol import TBinaryProtocol
+from thrift.transport import TTransport
+
+from e6data_python_connector.e6x_vector.ttypes import Chunk, Vector, VectorType
 from e6xdb.constants import ZONE
+from e6xdb.date_time_utils import floor_div, floor_mod
 
 _logger = logging.getLogger(__name__)
 
@@ -153,18 +159,103 @@ def read_values_from_array(query_columns_description: list, dis: DataInputStream
     return value_array
 
 
-def read_rows_from_batch(query_columns_description: list, dis: DataInputStream):
-    is_row_present = dis.read_byte()
-    if not is_row_present:
+def read_rows_from_chunk(query_columns_description: list, buffer):
+    # Create a transport and protocol instance for deserialization
+    transport = TTransport.TMemoryBuffer(buffer)
+    protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
+
+    # Create an instance of the Thrift struct and read from the protocol
+    chunk = Chunk()
+    chunk.read(protocol)
+
+    if chunk.size <= 0:
         return None
+
     rows = list()
-    while is_row_present == 1:
-        if is_row_present:
-            row = read_values_from_array(query_columns_description, dis)
-            rows.append(row)
-            #   if rows become 1000, break it
-            # if len(rows) == 1000:
-            #     _logger.info("Read Batch - Breaking the loop after 1000 records")
-            #     break
-        is_row_present = dis.read_byte()
+    columns = list()
+
+    for col, colName in enumerate(query_columns_description):
+        columns.append(get_column_from_chunk(chunk.vectors[col]))
+
+    for rowIndex in range(chunk.size):
+        value = list()
+        for colIndex, colName in enumerate(query_columns_description):
+            value.append(columns[colIndex][rowIndex])
+        rows.append(value)
+
     return rows
+
+
+def get_column_from_chunk(vector: Vector) -> list:
+    value_array = list()
+    d_type = vector.vectorType
+    try:
+        if d_type == VectorType.LONG:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                value_array.append(vector.data.int64Data.data[row] if not vector.isConstantVector else vector.data.numericConstantData.data)
+        elif d_type == VectorType.DATE:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                epoch_seconds = floor_div(vector.data.dateData.data[row] if not vector.isConstantVector else vector.data.dateConstantData.data, 1000_000)
+                zone_offset = pytz.timezone('UTC') if vector.zoneOffset == 'Z' else pytz.timezone(vector.zoneOffset)
+                date = datetime.fromtimestamp(epoch_seconds, zone_offset)
+                value_array.append(date.strftime("%Y-%m-%d"))
+        elif d_type == VectorType.DATETIME:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                epoch_micros = vector.data.timeData.data[row] if not vector.isConstantVector else vector.data.timeConstantData.data
+                epoch_seconds = floor_div(epoch_micros, 1000_000)
+                micros_of_the_day = floor_mod(epoch_micros, 1000_000)
+                zone_offset = pytz.timezone('UTC') if vector.zoneOffset == 'Z' else pytz.timezone(vector.zoneOffset)
+                date_time = datetime.fromtimestamp(epoch_seconds, zone_offset)
+                date_time = date_time + timedelta(microseconds=micros_of_the_day)
+                value_array.append(date_time.strftime("%Y-%m-%d %H:%M:%S"))
+        elif d_type == VectorType.STRING or d_type == VectorType.ARRAY or d_type == VectorType.MAP or d_type == VectorType.STRUCT:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                value_array.append(vector.data.varcharData.data[row] if not vector.isConstantVector else vector.data.varcharConstantData.data)
+        elif d_type == VectorType.DOUBLE:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                value_array.append(vector.data.float64Data.data[row] if not vector.isConstantVector else vector.data.numericDecimalConstantData.data)
+        elif d_type == VectorType.BINARY:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                value_array.append(vector.data.varcharData.data[row] if not vector.isConstantVector else vector.data.varcharConstantData.data)
+        elif d_type == VectorType.FLOAT:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                value_array.append(vector.data.float32Data.data[row] if not vector.isConstantVector else vector.data.numericDecimalConstantData.data)
+        elif d_type == VectorType.BOOLEAN:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                value_array.append(vector.data.boolData.data[row] if not vector.isConstantVector else vector.data.boolConstantData.data)
+        elif d_type == VectorType.INTEGER:
+            for row in range(vector.size):
+                if vector.nullSet[row]:
+                    value_array.append(None)
+                    continue
+                value_array.append(vector.data.int32Data.data[row] if not vector.isConstantVector else vector.data.numericConstantData.data)
+        else:
+            value_array.append(None)
+    except Exception as e:
+        _logger.error(e)
+        value_array.append('Failed to parse.')
+    return value_array

{e6data-python-connector-1.1.12 → e6data-python-connector-2.0.0}/e6data_python_connector/e6data_grpc.py

@@ -19,7 +19,7 @@ import grpc
 
 from e6data_python_connector.common import DBAPITypeObject, ParamEscaper, DBAPICursor
 from e6data_python_connector.constants import *
-from e6data_python_connector.datainputstream import DataInputStream, get_query_columns_info, read_rows_from_batch, \
+from e6data_python_connector.datainputstream import DataInputStream, get_query_columns_info, read_rows_from_chunk, \
     read_values_from_array
 from e6data_python_connector.server import e6x_engine_pb2_grpc, e6x_engine_pb2
 from e6data_python_connector.typeId import *
@@ -126,7 +126,7 @@ class Connection(object):
         self.catalog_name = catalog
 
         self._keepalive_timeout_ms = 900000
-        self._max_receive_message_length = 100 * 1024 * 1024  # mb
+        self._max_receive_message_length = -1
         self._max_send_message_length = 300 * 1024 * 1024  # mb
 
         if type(grpc_options) == dict:
@@ -490,32 +490,28 @@ class Cursor(DBAPICursor):
         get_next_result_batch_request = e6x_engine_pb2.GetNextResultBatchRequest(
             engineIP=self._engine_ip,
             sessionId=self.connection.get_session_id,
-            queryId=self._query_id,
-            asRowData=True
+            queryId=self._query_id
         )
         get_next_result_batch_response = client.getNextResultBatch(get_next_result_batch_request)
         buffer = get_next_result_batch_response.resultBatch
         if not self._is_metadata_updated:
             self.update_mete_data()
             self._is_metadata_updated = True
-        if not buffer:
+        if not buffer or len(buffer) == 0:
             return None
-        buffer = BytesIO(buffer)
-        dis = DataInputStream(buffer)
         # one batch retrieves the predefined set of rows
-        return read_rows_from_batch(self._query_columns_description, dis)
+        return read_rows_from_chunk(self._query_columns_description, buffer)
 
     def fetchall(self):
         return self._fetch_all()
 
-    def fetchmany(self, size=None):
+    def fetchmany(self, size: int = None):
         # _logger.info("fetching all from overriden method")
         if size is None:
             size = self.arraysize
         if self._data is None:
             self._data = list()
         while len(self._data) < size:
-            # _logger.info("fetching next batch from fetch many")
             rows = self.fetch_batch()
             if rows is None:
                 break
@@ -530,24 +526,10 @@ class Cursor(DBAPICursor):
 
     def fetchone(self):
         # _logger.info("fetch One returning the batch itself which is limited by predefined no.of rows")
-        rows_to_return = []
-        client = self.connection.client
-        get_next_result_row_request = e6x_engine_pb2.GetNextResultRowRequest(
-            engineIP=self._engine_ip,
-            sessionId=self.connection.get_session_id,
-            queryId=self._query_id
-        )
-        get_next_result_row_response = client.getNextResultRow(get_next_result_row_request)
-        buffer = get_next_result_row_response.resultRow
-        if not self._is_metadata_updated:
-            self.update_mete_data()
-            self._is_metadata_updated = True
-        if not buffer:
+        rows = self.fetchmany(1)
+        if rows is None or len(rows) == 0:
             return None
-        buffer = BytesIO(buffer)
-        dis = DataInputStream(buffer)
-        rows_to_return.append(read_values_from_array(self._query_columns_description, dis))
-        return rows_to_return
+        return rows
 
     def explain(self):
         explain_request = e6x_engine_pb2.ExplainRequest(

e6data-python-connector-2.0.0/e6data_python_connector/e6x_vector/__init__.py

@@ -0,0 +1 @@
+__all__ = ['ttypes', 'constants']

{e6data-python-connector-1.1.12/e6data_python_connector/server → e6data-python-connector-2.0.0/e6data_python_connector/e6x_vector}/constants.py

@@ -1,5 +1,5 @@
 #
-# Autogenerated by Thrift Compiler (0.18.1)
+# Autogenerated by Thrift Compiler (0.19.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #