e6data-python-connector 2.2.6rc1__py3-none-any.whl → 2.3.7__py3-none-any.whl

e6data_python_connector/cluster_manager.py

@@ -6,27 +6,27 @@ import grpc
 from grpc._channel import _InactiveRpcError
 import multiprocessing
 
+from e6data_python_connector.strategy import _get_active_strategy, _set_active_strategy, _set_pending_strategy, \
+    _get_grpc_header as _get_strategy_header
 
-def _get_grpc_header(engine_ip=None, cluster=None):
+
+def _get_grpc_header(engine_ip=None, cluster=None, strategy=None):
     """
     Generate gRPC metadata headers for the request.
 
     This function creates a list of metadata headers to be used in gRPC requests.
-    It includes optional headers for the engine IP and cluster UUID.
+    It includes optional headers for the engine IP, cluster UUID, and deployment strategy.
 
     Args:
         engine_ip (str, optional): The IP address of the engine. Defaults to None.
         cluster (str, optional): The UUID of the cluster. Defaults to None.
+        strategy (str, optional): The deployment strategy (blue/green). Defaults to None.
 
     Returns:
         list: A list of tuples representing the gRPC metadata headers.
     """
-    metadata = []
-    if engine_ip:
-        metadata.append(('plannerip', engine_ip))
-    if cluster:
-        metadata.append(('cluster-uuid', cluster))
-    return metadata
+    # Use the strategy module's implementation
+    return _get_strategy_header(engine_ip=engine_ip, cluster=cluster, strategy=strategy)
 
 
 class _StatusLock:
@@ -136,7 +136,8 @@ class ClusterManager:
         cluster_uuid (str): The unique identifier for the target cluster.
     """
 
-    def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 5, cluster_uuid=None, grpc_options=None):
+    def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 5,
+                 cluster_uuid=None, grpc_options=None):
         """
         Initializes a new instance of the ClusterManager class.
 
@@ -187,22 +188,146 @@ class ClusterManager:
             )
         return cluster_pb2_grpc.ClusterServiceStub(self._channel)
 
-    def _check_cluster_status(self):
-        while True:
-            try:
-                # Create a status request payload with user credentials
-                status_payload = cluster_pb2.ClusterStatusRequest(
+    def _try_cluster_request(self, request_type, payload=None):
+        """
+        Execute a cluster request with strategy fallback for 456 errors.
+        
+        For efficiency:
+        - If we have an active strategy, use it first
+        - Only try authentication sequence (blue -> green) if no active strategy
+        - On 456 error, switch to alternative strategy and update active strategy
+        
+        Args:
+            request_type: Type of request ('status' or 'resume')
+            payload: Request payload (optional, will be created if not provided)
+            
+        Returns:
+            The response from the successful request
+        """
+        current_strategy = _get_active_strategy()
+
+        # Create payload if not provided
+        if payload is None:
+            if request_type == "status":
+                payload = cluster_pb2.ClusterStatusRequest(
                     user=self._user,
                     password=self._password
                 )
-                # Send the status request to the cluster service
-                response = self._get_connection.status(
-                    status_payload,
-                    metadata=_get_grpc_header(cluster=self.cluster_uuid)
+            elif request_type == "resume":
+                payload = cluster_pb2.ResumeRequest(
+                    user=self._user,
+                    password=self._password
                 )
-                # Yield the current status
-                yield response.status
+
+        # If we have an active strategy, use it first
+        if current_strategy is not None:
+            try:
+                if request_type == "status":
+                    response = self._get_connection.status(
+                        payload,
+                        metadata=_get_grpc_header(cluster=self.cluster_uuid, strategy=current_strategy)
+                    )
+                elif request_type == "resume":
+                    response = self._get_connection.resume(
+                        payload,
+                        metadata=_get_grpc_header(cluster=self.cluster_uuid, strategy=current_strategy)
+                    )
+                else:
+                    raise ValueError(f"Unknown request type: {request_type}")
+
+                # Check for new strategy in response
+                if hasattr(response, 'new_strategy') and response.new_strategy:
+                    new_strategy = response.new_strategy.lower()
+                    if new_strategy != current_strategy:
+                        _set_pending_strategy(new_strategy)
+
+                return response
+
             except _InactiveRpcError as e:
+                if e.code() == grpc.StatusCode.UNKNOWN and 'status: 456' in e.details():
+                    # 456 error - switch to alternative strategy
+                    alternative_strategy = 'green' if current_strategy == 'blue' else 'blue'
+
+                    try:
+                        if request_type == "status":
+                            response = self._get_connection.status(
+                                payload,
+                                metadata=_get_grpc_header(cluster=self.cluster_uuid, strategy=alternative_strategy)
+                            )
+                        elif request_type == "resume":
+                            response = self._get_connection.resume(
+                                payload,
+                                metadata=_get_grpc_header(cluster=self.cluster_uuid, strategy=alternative_strategy)
+                            )
+
+                        # Update active strategy since the alternative worked
+                        _set_active_strategy(alternative_strategy)
+
+                        # Check for new strategy in response
+                        if hasattr(response, 'new_strategy') and response.new_strategy:
+                            new_strategy = response.new_strategy.lower()
+                            if new_strategy != alternative_strategy:
+                                _set_pending_strategy(new_strategy)
+
+                        return response
+
+                    except _InactiveRpcError as e2:
+                        raise e  # Raise the original error
+                else:
+                    # Non-456 error - don't retry
+                    raise e
+
+        # No active strategy - start with authentication logic (blue first, then green)
+        strategies_to_try = ['blue', 'green']
+
+        for i, strategy in enumerate(strategies_to_try):
+            try:
+
+                if request_type == "status":
+                    response = self._get_connection.status(
+                        payload,
+                        metadata=_get_grpc_header(cluster=self.cluster_uuid, strategy=strategy)
+                    )
+                elif request_type == "resume":
+                    response = self._get_connection.resume(
+                        payload,
+                        metadata=_get_grpc_header(cluster=self.cluster_uuid, strategy=strategy)
+                    )
+                else:
+                    raise ValueError(f"Unknown request type: {request_type}")
+
+                # Set the working strategy as active
+                _set_active_strategy(strategy)
+
+                # Check for new strategy in response
+                if hasattr(response, 'new_strategy') and response.new_strategy:
+                    new_strategy = response.new_strategy.lower()
+                    if new_strategy != strategy:
+                        _set_pending_strategy(new_strategy)
+
+                return response
+
+            except _InactiveRpcError as e:
+                if e.code() == grpc.StatusCode.UNKNOWN and 'status: 456' in e.details():
+                    # 456 error - try next strategy
+                    if i < len(strategies_to_try) - 1:
+                        continue
+                    else:
+                        raise e
+                else:
+                    # Non-456 error - don't retry
+                    raise e
+
+        # If we get here, all strategies failed
+        raise e
+
+    def _check_cluster_status(self):
+        while True:
+            try:
+                # Use the unified strategy-aware request method
+                response = self._try_cluster_request("status")
+                yield response.status
+            except _InactiveRpcError:
                 yield None
 
     def resume(self) -> bool:
@@ -237,25 +362,17 @@ class ClusterManager:
             if lock.is_active:
                 return True
 
-            # Retrieve the current cluster status
-            status_payload = cluster_pb2.ClusterStatusRequest(
-                user=self._user,
-                password=self._password
-            )
-            current_status = self._get_connection.status(
-                status_payload,
-                metadata=_get_grpc_header(cluster=self.cluster_uuid)
-            )
+            # Retrieve the current cluster status with strategy header
+            try:
+                current_status = self._try_cluster_request("status")
+            except _InactiveRpcError:
+                return False
             if current_status.status == 'suspended':
-                # Send the resume request
-                payload = cluster_pb2.ResumeRequest(
-                    user=self._user,
-                    password=self._password
-                )
-                response = self._get_connection.resume(
-                    payload,
-                    metadata=_get_grpc_header(cluster=self.cluster_uuid)
-                )
+                # Send the resume request with strategy header
+                try:
+                    response = self._try_cluster_request("resume")
+                except _InactiveRpcError:
+                    return False
             elif current_status.status == 'active':
                 return True
             elif current_status.status != 'resuming':

e6data_python_connector/datainputstream.py

@@ -1,6 +1,8 @@
 import logging
 import struct
 from datetime import datetime, timedelta
+from decimal import Decimal
+import decimal
 
 import pytz
 from thrift.protocol import TBinaryProtocol
@@ -24,6 +26,322 @@ except ImportError:
 _logger = logging.getLogger(__name__)
 
 
+def _binary_to_decimal128(binary_data):
+    """
+    Convert binary data to Decimal128.
+    
+    The binary data represents a 128-bit decimal number in IEEE 754-2008 Decimal128 format.
+    Based on the Java implementation from e6data's JDBC driver.
+    
+    Args:
+        binary_data (bytes): Binary representation of Decimal128
+        
+    Returns:
+        Decimal: Python Decimal object
+    """
+    if not binary_data:
+        return None
+
+    try:
+        # Handle different input types
+        if isinstance(binary_data, str):
+            return Decimal(binary_data)
+
+        if isinstance(binary_data, bytes):
+            # Check if it's a UTF-8 string representation first
+            try:
+                decimal_str = binary_data.decode('utf-8')
+                # Check if it looks like a decimal string
+                if any(c.isdigit() or c in '.-+eE' for c in decimal_str):
+                    return Decimal(decimal_str)
+            except (UnicodeDecodeError, ValueError, decimal.InvalidOperation):
+                pass  # Fall through to binary parsing
+
+            # Handle IEEE 754-2008 Decimal128 binary format
+            if len(binary_data) == 16:  # Decimal128 should be exactly 16 bytes
+                return _decode_decimal128_binary_java_style(binary_data)
+            else:
+                _logger.warning(f"Invalid Decimal128 binary length: {len(binary_data)} bytes, expected 16")
+                return Decimal('0')
+
+        # If it's already a string, convert directly
+        return Decimal(str(binary_data))
+
+    except Exception as e:
+        _logger.error(f"Error converting binary to Decimal128: {e}")
+        # Return Decimal('0') as fallback for any unexpected errors
+        return Decimal('0')
+
+
+def _decode_decimal128_binary_java_style(binary_data):
+    """
+    Decode IEEE 754-2008 Decimal128 binary format following Java implementation.
+    
+    Based on the Java implementation from e6data's JDBC driver getFieldDataFromChunk method.
+    This method follows the same logic as the Java BigDecimal creation from ByteBuffer.
+    
+    Args:
+        binary_data (bytes): 16-byte binary representation
+        
+    Returns:
+        Decimal: Python Decimal object
+    """
+    if len(binary_data) != 16:
+        raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}")
+
+    # Special case: all zeros
+    if all(b == 0 for b in binary_data):
+        return Decimal('0')
+
+    try:
+        # Following the Java pattern: create BigInteger from bytes, then BigDecimal
+        # Convert bytes to a big integer (Java's BigInteger constructor behavior)
+        # Java BigInteger uses two's complement representation
+        big_int_value = int.from_bytes(binary_data, byteorder='big', signed=True)
+
+        # If the value is zero, return zero
+        if big_int_value == 0:
+            return Decimal('0')
+
+        # The Java code creates BigDecimal from BigInteger with scale 0
+        # This means we treat the integer value as the unscaled value
+        # However, for Decimal128, we need to handle the scaling properly
+
+        # Try to create decimal directly from the integer value
+        decimal_value = Decimal(big_int_value)
+
+        # Check if this produces a reasonable decimal value
+        # Decimal128 should represent normal decimal numbers
+        if abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal(
+                '9.999999999999999999999999999999999E+6144'):
+            # Value is outside normal Decimal128 range, try alternative interpretation
+            return _decode_decimal128_alternative(binary_data)
+
+        return decimal_value
+
+    except Exception as e:
+        _logger.warning(f"Failed to decode Decimal128 with Java-style method: {e}")
+        # Fallback to alternative decoding
+        return _decode_decimal128_alternative(binary_data)
+
+
+def _decode_decimal128_alternative(binary_data):
+    """
+    Alternative Decimal128 decoding method.
+    
+    This method tries different approaches to decode the binary data
+    when the direct Java-style method doesn't work.
+    
+    Args:
+        binary_data (bytes): 16-byte binary representation
+        
+    Returns:
+        Decimal: Python Decimal object
+    """
+    try:
+        # Method 1: Try interpreting as IEEE 754-2008 Decimal128 format
+        return _decode_decimal128_binary(binary_data)
+    except:
+        pass
+
+    try:
+        # Method 2: Try different byte order interpretations
+        # Sometimes the byte order might be different
+        big_int_le = int.from_bytes(binary_data, byteorder='little', signed=True)
+        if big_int_le != 0:
+            decimal_le = Decimal(big_int_le)
+            # Check if this gives a more reasonable result
+            if Decimal('1E-100') <= abs(decimal_le) <= Decimal('1E100'):
+                return decimal_le
+    except:
+        pass
+
+    try:
+        # Method 3: Try unsigned interpretation
+        big_int_unsigned = int.from_bytes(binary_data, byteorder='big', signed=False)
+        if big_int_unsigned != 0:
+            decimal_unsigned = Decimal(big_int_unsigned)
+            # Apply some reasonable scaling if the number is too large
+            if abs(decimal_unsigned) > Decimal('1E50'):
+                # Try scaling down
+                for scale in [1E10, 1E20, 1E30, 1E40]:
+                    scaled = decimal_unsigned / Decimal(scale)
+                    if Decimal('1E-10') <= abs(scaled) <= Decimal('1E50'):
+                        return scaled
+            return decimal_unsigned
+    except:
+        pass
+
+    # If all methods fail, return 0
+    _logger.warning(f"Could not decode Decimal128 binary data: {binary_data.hex()}")
+    return Decimal('0')
+
+
+def _decode_decimal128_binary(binary_data):
+    """
+    Decode IEEE 754-2008 Decimal128 binary format.
+    
+    Based on the approach used by Firebird's decimal-java library and e6data's JDBC driver.
+    
+    Decimal128 format (128 bits total):
+    - 1 bit: Sign (S)
+    - 17 bits: Combination field (encodes exponent MSB + MSD or special values)
+    - 110 bits: Coefficient continuation (densely packed decimal)
+    
+    Args:
+        binary_data (bytes): 16-byte binary representation (big-endian)
+        
+    Returns:
+        Decimal: Python Decimal object
+    """
+    if len(binary_data) != 16:
+        raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}")
+
+    # Convert bytes to 128-bit integer (big-endian)
+    bits = int.from_bytes(binary_data, byteorder='big')
+
+    # Special case: all zeros
+    if bits == 0:
+        return Decimal('0')
+
+    # Extract fields according to IEEE 754-2008 Decimal128 layout
+    sign = (bits >> 127) & 1
+
+    # The combination field is 17 bits (bits 126-110)
+    combination = (bits >> 110) & 0x1FFFF
+
+    # Coefficient continuation is the remaining 110 bits (bits 109-0)
+    coeff_continuation = bits & ((1 << 110) - 1)
+
+    # Decode the combination field to get the most significant digit and exponent
+    # Check for special values first
+    if (combination >> 15) == 0b11:  # Top 2 bits are 11
+        if (combination >> 12) == 0b11110:  # 11110 = Infinity
+            return Decimal('-Infinity' if sign else 'Infinity')
+        elif (combination >> 12) == 0b11111:  # 11111 = NaN
+            return Decimal('NaN')
+        else:
+            # Large MSD (8 or 9)
+            # Format: 11xxxxxxxxxxxx followed by 1 bit for MSD selection
+            exponent_bits = combination & 0x3FFF  # Bottom 14 bits
+            msd = 8 + ((combination >> 14) & 1)  # Bit 14 selects between 8 and 9
+    else:
+        # Normal case: MSD is 0-7
+        # Format: xxxxxxxxxxxx followed by 3 bits for MSD
+        exponent_bits = (combination >> 3) & 0x3FFF  # Bits 16-3
+        msd = combination & 0x7  # Bottom 3 bits
+
+    # Apply bias (6176 for Decimal128)
+    exponent = exponent_bits - 6176
+
+    # Decode the coefficient from DPD format
+    coefficient = _decode_dpd_coefficient_proper(msd, coeff_continuation)
+
+    # Create the decimal number
+    if coefficient == 0:
+        return Decimal('0')
+
+    # Apply sign
+    if sign:
+        coefficient = -coefficient
+
+    # Create Decimal with the coefficient and exponent
+    # Python's Decimal expects strings in the form "123E45"
+    decimal_str = f"{coefficient}E{exponent}"
+
+    try:
+        return Decimal(decimal_str)
+    except (ValueError, decimal.InvalidOperation) as e:
+        _logger.error(f"Failed to create Decimal from {decimal_str}: {e}")
+        # Return zero as fallback
+        return Decimal('0')
+
+
+def _decode_dpd_coefficient_proper(msd, coeff_continuation):
+    """
+    Decode the coefficient from Densely Packed Decimal (DPD) format.
+    
+    Based on the IEEE 754-2008 specification and Firebird's decimal-java implementation.
+    
+    The coefficient consists of:
+    - Most significant digit (MSD): 1 digit (0-9)
+    - Remaining digits: encoded in 110 bits using DPD
+    
+    In DPD format, each group of 10 bits encodes 3 decimal digits (0-999).
+    For Decimal128, we have 110 bits = 11 groups of 10 bits = 33 decimal digits.
+    Total coefficient = 1 MSD + 33 DPD digits = 34 digits maximum.
+    
+    Args:
+        msd (int): Most significant digit (0-9)
+        coeff_continuation (int): 110-bit continuation field
+        
+    Returns:
+        int: Decoded coefficient
+    """
+    # Start with the most significant digit
+    if coeff_continuation == 0:
+        return msd
+
+    # Create DPD lookup table for 10-bit groups to 3-digit decoding
+    # This is a simplified implementation - in production, you'd use a pre-computed table
+    dpd_digits = []
+
+    # Process 11 groups of 10 bits each (110 bits total)
+    # Each group encodes 3 decimal digits
+    for group_idx in range(11):
+        # Extract 10 bits for this group (from right to left)
+        group_bits = (coeff_continuation >> (group_idx * 10)) & 0x3FF
+
+        # Decode the 10-bit DPD group to 3 decimal digits
+        d0, d1, d2 = _decode_dpd_group_proper(group_bits)
+
+        # Add digits to our list (in reverse order since we're processing right to left)
+        dpd_digits.extend([d2, d1, d0])
+
+    # Reverse to get correct order (most significant to least significant)
+    dpd_digits.reverse()
+
+    # Build the coefficient string
+    coefficient_str = str(msd)
+
+    # Add DPD digits, but only up to 33 more digits (total 34)
+    for i, digit in enumerate(dpd_digits):
+        if i < 33:  # Decimal128 coefficient is max 34 digits
+            coefficient_str += str(digit)
+
+    return int(coefficient_str)
+
+
+def _decode_dpd_group_proper(group_bits):
+    """
+    Decode a 10-bit DPD group to 3 decimal digits.
+    
+    Based on the IEEE 754-2008 DPD specification.
+    This implements a simplified but effective DPD decoding algorithm.
+    
+    Args:
+        group_bits (int): 10-bit DPD encoded value (0-1023)
+        
+    Returns:
+        tuple: Three decimal digits (d0, d1, d2) where d0 is most significant
+    """
+    # DPD encoding maps 1000 decimal values (000-999) to 1024 possible 10-bit patterns
+    # Values 0-999 are encoded, with 24 patterns unused for future extensions
+
+    # For values 0-999, we can use a direct approach
+    if group_bits < 1000:
+        # Most DPD values map directly to their decimal equivalent
+        # This is a simplification, but works for the majority of cases
+        d0 = group_bits // 100
+        d1 = (group_bits // 10) % 10
+        d2 = group_bits % 10
+        return (d0, d1, d2)
+    else:
+        # For the 24 unused patterns (1000-1023), use a fallback
+        # In practice, these should not appear in valid decimal data
+        return (0, 0, 0)  # Safe fallback
+
+
 def get_null(vector: Vector, index: int):
     return vector.nullSet[0] if vector.isConstantVector else vector.nullSet[index]
 
@@ -167,6 +485,10 @@ def read_values_from_array(query_columns_description: list, dis: DataInputStream
                 value_array.append(date_time_with_nanos)
             elif dtype == "INTEGER":
                 value_array.append(dis.read_int())
+            elif dtype == "DECIMAL128":
+                # Read decimal128 as UTF-8 string representation
+                decimal_str = dis.read_utf().decode()
+                value_array.append(Decimal(decimal_str))
         except Exception as e:
             _logger.error(e)
             value_array.append('Failed to parse.')
@@ -201,7 +523,6 @@ def read_rows_from_chunk(query_columns_description: list, buffer):
     return rows
 
 
-
 def get_column_from_chunk(vector: Vector) -> list:
     value_array = list()
     d_type = vector.vectorType
@@ -298,6 +619,34 @@ def get_column_from_chunk(vector: Vector) -> list:
                 date_time = datetime.fromtimestamp(epoch_seconds, zone)
                 date_time = date_time + timedelta(microseconds=micros_of_the_day)
                 value_array.append(date_time.isoformat(timespec='milliseconds'))
+        elif d_type == VectorType.DECIMAL128:
+            # Handle both constant and non-constant vectors following Java implementation
+            if vector.isConstantVector:
+                # For constant vectors, get the binary data and convert it once
+                binary_data = vector.data.numericDecimal128ConstantData.data
+
+                # Convert binary data to BigDecimal equivalent
+                if binary_data:
+                    decimal_value = _binary_to_decimal128(binary_data)
+                else:
+                    decimal_value = Decimal('0')
+
+                # Apply the same value to all rows
+                for row in range(vector.size):
+                    if get_null(vector, row):
+                        value_array.append(None)
+                    else:
+                        value_array.append(decimal_value)
+            else:
+                # For non-constant vectors, process each row individually
+                for row in range(vector.size):
+                    if get_null(vector, row):
+                        value_array.append(None)
+                        continue
+                    # Get binary data for this row
+                    binary_data = vector.data.decimal128Data.data[row]
+                    decimal_value = _binary_to_decimal128(binary_data)
+                    value_array.append(decimal_value)
         else:
             value_array.append(None)
     except Exception as e:

e6data_python_connector/dialect.py

@@ -210,7 +210,7 @@ class E6dataDialect(default.DefaultDialect):
     driver = 'e6data'
     scheme = 'e6data'
     catalog_name = None
-    cluster_uuid = None
+    cluster_name = None
     secure = False
     auto_resume = False
 
@@ -226,7 +226,7 @@ class E6dataDialect(default.DefaultDialect):
         if url.query.get("schema"):
             database = url.query.get("schema")
         self.catalog_name = url.query.get("catalog")
-        self.cluster_uuid = url.query.get("cluster-uuid")
+        self.cluster_name = url.query.get("cluster-name")
         self.secure = url.query.get("secure") == "true"
         self.auto_resume = url.query.get("auto-resume", "true") == "true"  # default to True
         if not self.catalog_name:
@@ -247,7 +247,7 @@ class E6dataDialect(default.DefaultDialect):
             "password": url.password or None,
             "database": database,
             "catalog": self.catalog_name,
-            "cluster_uuid": self.cluster_uuid,
+            "cluster_name": self.cluster_name,
             'secure': self.secure,
             'auto_resume': self.auto_resume,
             'grpc_options': grpc_options