e6data-python-connector 2.3.10rc7__tar.gz → 2.3.11__tar.gz

{e6data_python_connector-2.3.10rc7/e6data_python_connector.egg-info → e6data_python_connector-2.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: e6data-python-connector
-Version: 2.3.10rc7
+Version: 2.3.11
 Summary: Client for the e6data distributed SQL Engine.
 Home-page: https://github.com/e6x-labs/e6data-python-connector
 Author: e6data, Inc.
@@ -39,7 +39,7 @@ Dynamic: summary
 
 # e6data Python Connector
 
-![version](https://img.shields.io/badge/version-2.3.10rc7-blue.svg)
+![version](https://img.shields.io/badge/version-2.3.11-blue.svg)
 
 ## Introduction
 
@@ -124,6 +124,7 @@ The `Connection` class supports the following parameters:
 | `secure` | bool | No | False | Enable SSL/TLS for secure connections |
 | `auto_resume` | bool | No | True | Automatically resume cluster if suspended |
 | `grpc_options` | dict | No | None | Additional gRPC configuration options |
+| `debug` | bool | No | False | Enable debug logging for troubleshooting |
 
 #### Secure Connection Example
 
@@ -637,4 +638,58 @@ conn.close()
 - Graceful connection recovery and retry logic
 - Blue-green deployment support with automatic failover
 
+## Debugging and Troubleshooting
+
+### Enable Debug Mode
+
+Enable comprehensive debugging to troubleshoot connection and query issues:
+
+```python
+from e6data_python_connector import Connection
+
+conn = Connection(
+    host=host,
+    port=port,
+    username=username,
+    password=password,
+    database=database,
+    debug=True  # Enable debug logging
+)
+```
+
+When `debug=True`, the following features are enabled:
+- Python logging at DEBUG level for all operations
+- Blue-green strategy transition logging
+- Connection lifecycle logging
+- Query execution detailed logging
+
+### gRPC Network Tracing
+
+For low-level gRPC network debugging (HTTP/2 frames, TCP events), set environment variables **before** running your Python script:
+
+```bash
+# Enable gRPC network tracing
+export GRPC_VERBOSITY=DEBUG
+export GRPC_TRACE=client_channel,http2
+
+# For comprehensive tracing
+export GRPC_TRACE=api,call_error,channel,client_channel,connectivity_state,http,http2_stream,tcp,transport_security
+
+# Run your script
+python your_script.py
+```
+
+**Note**: These environment variables must be set before Python starts, as the gRPC C++ core reads them at module import time.
+
+### Common Issues and Solutions
+
+| Issue | Solution |
+|-------|----------|
+| Connection timeout | Check network connectivity, firewall rules, and ensure port 80/443 is open |
+| Authentication failure | Verify username (email) and access token are correct |
+| 503 Service Unavailable | Cluster may be suspended; enable `auto_resume=True` |
+| 456 Strategy Error | Automatic blue-green failover will handle this |
+| Memory issues with large results | Use `fetchall_buffer()` instead of `fetchall()` |
+| gRPC message size errors | Configure `grpc_options` with appropriate message size limits |
+
 See [TECH_DOC.md](TECH_DOC.md) for detailed technical documentation.

{e6data_python_connector-2.3.10rc7 → e6data_python_connector-2.3.11}/README.md

@@ -1,6 +1,6 @@
 # e6data Python Connector
 
-![version](https://img.shields.io/badge/version-2.3.10rc7-blue.svg)
+![version](https://img.shields.io/badge/version-2.3.11-blue.svg)
 
 ## Introduction
 
@@ -85,6 +85,7 @@ The `Connection` class supports the following parameters:
 | `secure` | bool | No | False | Enable SSL/TLS for secure connections |
 | `auto_resume` | bool | No | True | Automatically resume cluster if suspended |
 | `grpc_options` | dict | No | None | Additional gRPC configuration options |
+| `debug` | bool | No | False | Enable debug logging for troubleshooting |
 
 #### Secure Connection Example
 
@@ -598,4 +599,58 @@ conn.close()
 - Graceful connection recovery and retry logic
 - Blue-green deployment support with automatic failover
 
+## Debugging and Troubleshooting
+
+### Enable Debug Mode
+
+Enable comprehensive debugging to troubleshoot connection and query issues:
+
+```python
+from e6data_python_connector import Connection
+
+conn = Connection(
+    host=host,
+    port=port,
+    username=username,
+    password=password,
+    database=database,
+    debug=True  # Enable debug logging
+)
+```
+
+When `debug=True`, the following features are enabled:
+- Python logging at DEBUG level for all operations
+- Blue-green strategy transition logging
+- Connection lifecycle logging
+- Query execution detailed logging
+
+### gRPC Network Tracing
+
+For low-level gRPC network debugging (HTTP/2 frames, TCP events), set environment variables **before** running your Python script:
+
+```bash
+# Enable gRPC network tracing
+export GRPC_VERBOSITY=DEBUG
+export GRPC_TRACE=client_channel,http2
+
+# For comprehensive tracing
+export GRPC_TRACE=api,call_error,channel,client_channel,connectivity_state,http,http2_stream,tcp,transport_security
+
+# Run your script
+python your_script.py
+```
+
+**Note**: These environment variables must be set before Python starts, as the gRPC C++ core reads them at module import time.
+
+### Common Issues and Solutions
+
+| Issue | Solution |
+|-------|----------|
+| Connection timeout | Check network connectivity, firewall rules, and ensure port 80/443 is open |
+| Authentication failure | Verify username (email) and access token are correct |
+| 503 Service Unavailable | Cluster may be suspended; enable `auto_resume=True` |
+| 456 Strategy Error | Automatic blue-green failover will handle this |
+| Memory issues with large results | Use `fetchall_buffer()` instead of `fetchall()` |
+| gRPC message size errors | Configure `grpc_options` with appropriate message size limits |
+
 See [TECH_DOC.md](TECH_DOC.md) for detailed technical documentation.

{e6data_python_connector-2.3.10rc7 → e6data_python_connector-2.3.11}/e6data_python_connector/cluster_manager.py

@@ -1,3 +1,4 @@
+import logging
 import threading
 import time
 import e6data_python_connector.cluster_server.cluster_pb2 as cluster_pb2
@@ -6,6 +7,8 @@ import grpc
 from grpc._channel import _InactiveRpcError
 import multiprocessing
 
+logger = logging.getLogger(__name__)
+
 from e6data_python_connector.strategy import _get_active_strategy, _set_active_strategy, _set_pending_strategy, \
     _get_grpc_header as _get_strategy_header
 
@@ -137,7 +140,7 @@ class ClusterManager:
     """
 
     def __init__(self, host: str, port: int, user: str, password: str, secure_channel: bool = False, timeout=60 * 5,
-                 cluster_uuid=None, grpc_options=None):
+                 cluster_uuid=None, grpc_options=None, debug=False):
         """
         Initializes a new instance of the ClusterManager class.
 
@@ -152,6 +155,7 @@ class ClusterManager:
                 defaults to 5 minutes.
             cluster_uuid (str, optional): The unique identifier for the target cluster;
                 defaults to None.
+            debug (bool, optional): Enable debug logging; defaults to False.
         """
 
         self._host = host
@@ -164,6 +168,7 @@ class ClusterManager:
         self._grpc_options = grpc_options
         if grpc_options is None:
             self._grpc_options = dict()
+        self._debug = debug
 
     @property
     def _get_connection(self):
@@ -191,20 +196,22 @@ class ClusterManager:
     def _try_cluster_request(self, request_type, payload=None):
         """
         Execute a cluster request with strategy fallback for 456 errors.
-        
+
         For efficiency:
         - If we have an active strategy, use it first
         - Only try authentication sequence (blue -> green) if no active strategy
         - On 456 error, switch to alternative strategy and update active strategy
-        
+
         Args:
             request_type: Type of request ('status' or 'resume')
             payload: Request payload (optional, will be created if not provided)
-            
+
         Returns:
             The response from the successful request
         """
         current_strategy = _get_active_strategy()
+        if self._debug:
+            logger.info(f"Attempting {request_type} request with current strategy: {current_strategy}")
 
         # Create payload if not provided
         if payload is None:
@@ -222,6 +229,8 @@ class ClusterManager:
         # If we have an active strategy, use it first
         if current_strategy is not None:
             try:
+                if self._debug:
+                    logger.info(f"Executing {request_type} with strategy '{current_strategy}'")
                 if request_type == "status":
                     response = self._get_connection.status(
                         payload,
@@ -239,16 +248,24 @@ class ClusterManager:
                 if hasattr(response, 'new_strategy') and response.new_strategy:
                     new_strategy = response.new_strategy.lower()
                     if new_strategy != current_strategy:
+                        if self._debug:
+                            logger.info(f"Server indicated strategy transition from '{current_strategy}' to '{new_strategy}'")
                         _set_pending_strategy(new_strategy)
 
+                if self._debug:
+                    logger.info(f"{request_type} request successful with strategy '{current_strategy}'")
                 return response
 
             except _InactiveRpcError as e:
                 if e.code() == grpc.StatusCode.UNKNOWN and 'status: 456' in e.details():
                     # 456 error - switch to alternative strategy
                     alternative_strategy = 'green' if current_strategy == 'blue' else 'blue'
+                    if self._debug:
+                        logger.info(f"Received 456 error with strategy '{current_strategy}', switching to '{alternative_strategy}'")
 
                     try:
+                        if self._debug:
+                            logger.info(f"Retrying {request_type} with alternative strategy '{alternative_strategy}'")
                         if request_type == "status":
                             response = self._get_connection.status(
                                 payload,
@@ -261,27 +278,39 @@ class ClusterManager:
                             )
 
                         # Update active strategy since the alternative worked
+                        if self._debug:
+                            logger.info(f"Successfully switched to strategy '{alternative_strategy}'")
                         _set_active_strategy(alternative_strategy)
 
                         # Check for new strategy in response
                         if hasattr(response, 'new_strategy') and response.new_strategy:
                             new_strategy = response.new_strategy.lower()
                             if new_strategy != alternative_strategy:
+                                if self._debug:
+                                    logger.info(f"Server indicated future strategy transition to '{new_strategy}'")
                                 _set_pending_strategy(new_strategy)
 
                         return response
 
                     except _InactiveRpcError as e2:
+                        if self._debug:
+                            logger.error(f"Failed with alternative strategy '{alternative_strategy}': {e2}")
                         raise e  # Raise the original error
                 else:
                     # Non-456 error - don't retry
+                    if self._debug:
+                        logger.info(f"Non-456 error received, not retrying: {e}")
                     raise e
 
         # No active strategy - start with authentication logic (blue first, then green)
         strategies_to_try = ['blue', 'green']
+        if self._debug:
+            logger.info("No active strategy found, trying authentication sequence: blue -> green")
 
         for i, strategy in enumerate(strategies_to_try):
             try:
+                if self._debug:
+                    logger.info(f"Trying {request_type} with strategy '{strategy}' (attempt {i+1}/{len(strategies_to_try)})")
 
                 if request_type == "status":
                     response = self._get_connection.status(
@@ -297,12 +326,16 @@ class ClusterManager:
                     raise ValueError(f"Unknown request type: {request_type}")
 
                 # Set the working strategy as active
+                if self._debug:
+                    logger.info(f"Successfully authenticated with strategy '{strategy}'")
                 _set_active_strategy(strategy)
 
                 # Check for new strategy in response
                 if hasattr(response, 'new_strategy') and response.new_strategy:
                     new_strategy = response.new_strategy.lower()
                     if new_strategy != strategy:
+                        if self._debug:
+                            logger.info(f"Server indicated strategy transition to '{new_strategy}'")
                         _set_pending_strategy(new_strategy)
 
                 return response
@@ -311,14 +344,22 @@ class ClusterManager:
                 if e.code() == grpc.StatusCode.UNKNOWN and 'status: 456' in e.details():
                     # 456 error - try next strategy
                     if i < len(strategies_to_try) - 1:
+                        if self._debug:
+                            logger.info(f"Strategy '{strategy}' returned 456 error, trying next strategy")
                         continue
                     else:
+                        if self._debug:
+                            logger.error(f"All strategies failed with 456 errors")
                         raise e
                 else:
                     # Non-456 error - don't retry
+                    if self._debug:
+                        logger.error(f"Strategy '{strategy}' failed with non-456 error: {e}")
                     raise e
 
         # If we get here, all strategies failed
+        if self._debug:
+            logger.error("All authentication strategies exhausted")
         raise e
 
     def _check_cluster_status(self):
@@ -326,8 +367,12 @@ class ClusterManager:
             try:
                 # Use the unified strategy-aware request method
                 response = self._try_cluster_request("status")
+                if self._debug:
+                    logger.info(f"Cluster status check returned: {response.status}")
                 yield response.status
-            except _InactiveRpcError:
+            except _InactiveRpcError as e:
+                if self._debug:
+                    logger.warning(f"Status check failed with error: {e}")
                 yield None
 
     def resume(self) -> bool:
@@ -357,38 +402,77 @@ class ClusterManager:
             - The operation is subject to the `_timeout` threshold;
               if the timeout expires, the method returns False.
         """
+        if self._debug:
+            logger.info(f"Starting auto-resume for cluster {self.cluster_uuid} at {self._host}:{self._port}")
 
         with status_lock as lock:
             if lock.is_active:
+                if self._debug:
+                    logger.info("Lock is already active, cluster appears to be running")
                 return True
 
             # Retrieve the current cluster status with strategy header
+            if self._debug:
+                logger.info("Checking current cluster status")
             try:
                 current_status = self._try_cluster_request("status")
-            except _InactiveRpcError:
+                if self._debug:
+                    logger.info(f"Current cluster status: {current_status.status}")
+            except _InactiveRpcError as e:
+                if self._debug:
+                    logger.error(f"Failed to get cluster status: {e}")
                 return False
+
             if current_status.status == 'suspended':
                 # Send the resume request with strategy header
+                if self._debug:
+                    logger.info("Cluster is suspended, sending resume request")
                 try:
                     response = self._try_cluster_request("resume")
-                except _InactiveRpcError:
+                    if self._debug:
+                        logger.info("Resume request sent successfully")
+                except _InactiveRpcError as e:
+                    if self._debug:
+                        logger.error(f"Failed to send resume request: {e}")
                     return False
             elif current_status.status == 'active':
+                if self._debug:
+                    logger.info("Cluster is already active, no action needed")
                 return True
             elif current_status.status != 'resuming':
                 """
-                 If cluster cannot be resumed due to its current state, 
+                 If cluster cannot be resumed due to its current state,
                  or already in a process of resuming, terminate the operation.
                  """
+                if self._debug:
+                    logger.warning(f"Cluster is in state '{current_status.status}' which cannot be resumed")
                 return False
 
+            if self._debug:
+                logger.info("Monitoring cluster status until it becomes active...")
+            check_count = 0
             for status in self._check_cluster_status():
+                check_count += 1
                 if status == 'active':
+                    if self._debug:
+                        logger.info(f"Cluster became active after {check_count} status checks")
                     return True
-                elif status == 'failed' or time.time() > self._timeout:
+                elif status == 'failed':
+                    if self._debug:
+                        logger.error(f"Cluster failed to resume after {check_count} status checks")
+                    return False
+                elif time.time() > self._timeout:
+                    if self._debug:
+                        logger.error(f"Resume operation timed out after {check_count} status checks")
                     return False
+
+                if self._debug:
+                    logger.info(f"Status check #{check_count}: {status}, waiting 5 seconds before next check")
                 # Wait for 5 seconds before the next status check
                 time.sleep(5)
+
+            if self._debug:
+                logger.warning("Resume operation completed without reaching active state")
             return False
 
     def suspend(self):

{e6data_python_connector-2.3.10rc7 → e6data_python_connector-2.3.11}/e6data_python_connector/datainputstream.py

@@ -26,16 +26,18 @@ except ImportError:
 _logger = logging.getLogger(__name__)
 
 
-def _binary_to_decimal128(binary_data):
+def _binary_to_decimal128(binary_data, scale=None):
     """
     Convert binary data to Decimal128.
-    
+
     The binary data represents a 128-bit decimal number in IEEE 754-2008 Decimal128 format.
     Based on the Java implementation from e6data's JDBC driver.
-    
+
     Args:
         binary_data (bytes): Binary representation of Decimal128
-        
+        scale (int, optional): Scale parameter for decimal precision. If None,
+                               attempts to decode from IEEE format or defaults to 0.
+
     Returns:
         Decimal: Python Decimal object
     """
@@ -59,7 +61,7 @@ def _binary_to_decimal128(binary_data):
 
             # Handle IEEE 754-2008 Decimal128 binary format
             if len(binary_data) == 16:  # Decimal128 should be exactly 16 bytes
-                return _decode_decimal128_binary_java_style(binary_data)
+                return _decode_decimal128_binary_java_style(binary_data, scale)
             else:
                 _logger.warning(f"Invalid Decimal128 binary length: {len(binary_data)} bytes, expected 16")
                 return Decimal('0')
@@ -73,16 +75,17 @@ def _binary_to_decimal128(binary_data):
         return Decimal('0')
 
 
-def _decode_decimal128_binary_java_style(binary_data):
+def _decode_decimal128_binary_java_style(binary_data, scale=None):
     """
     Decode IEEE 754-2008 Decimal128 binary format following Java implementation.
-    
+
     Based on the Java implementation from e6data's JDBC driver getFieldDataFromChunk method.
     This method follows the same logic as the Java BigDecimal creation from ByteBuffer.
-    
+
     Args:
         binary_data (bytes): 16-byte binary representation
-        
+        scale (int, optional): Scale parameter for decimal precision
+
     Returns:
         Decimal: Python Decimal object
     """
@@ -103,17 +106,21 @@ def _decode_decimal128_binary_java_style(binary_data):
         if big_int_value == 0:
             return Decimal('0')
 
-        # The Java code creates BigDecimal from BigInteger with scale 0
-        # This means we treat the integer value as the unscaled value
-        # However, for Decimal128, we need to handle the scaling properly
-
-        # Try to create decimal directly from the integer value
-        decimal_value = Decimal(big_int_value)
+        # The Java code creates BigDecimal from BigInteger with the provided scale
+        # If scale is provided, use it to create the properly scaled decimal
+        if scale is not None:
+            # Create decimal with the specified scale (like Java's BigDecimal constructor)
+            # This treats big_int_value as the unscaled value
+            decimal_value = Decimal(big_int_value) / (Decimal(10) ** scale)
+        else:
+            # Fallback: try to create decimal directly from the integer value
+            decimal_value = Decimal(big_int_value)
 
         # Check if this produces a reasonable decimal value
         # Decimal128 should represent normal decimal numbers
-        if abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal(
-                '9.999999999999999999999999999999999E+6144'):
+        # Only check range if scale was not provided (backward compatibility)
+        if scale is None and (abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal(
+                '9.999999999999999999999999999999999E+6144')):
             # Value is outside normal Decimal128 range, try alternative interpretation
             return _decode_decimal128_alternative(binary_data)
 
@@ -624,10 +631,12 @@ def get_column_from_chunk(vector: Vector) -> list:
             if vector.isConstantVector:
                 # For constant vectors, get the binary data and convert it once
                 binary_data = vector.data.numericDecimal128ConstantData.data
+                # Get scale with backward compatibility for older engines
+                scale = getattr(vector.data.numericDecimal128ConstantData, 'scale', None)
 
                 # Convert binary data to BigDecimal equivalent
                 if binary_data:
-                    decimal_value = _binary_to_decimal128(binary_data)
+                    decimal_value = _binary_to_decimal128(binary_data, scale)
                 else:
                     decimal_value = Decimal('0')
 
@@ -639,13 +648,16 @@ def get_column_from_chunk(vector: Vector) -> list:
                         value_array.append(decimal_value)
             else:
                 # For non-constant vectors, process each row individually
+                # Get scale from decimal128Data (with backward compatibility)
+                scale = getattr(vector.data.decimal128Data, 'scale', None)
+
                 for row in range(vector.size):
                     if get_null(vector, row):
                         value_array.append(None)
                         continue
                     # Get binary data for this row
                     binary_data = vector.data.decimal128Data.data[row]
-                    decimal_value = _binary_to_decimal128(binary_data)
+                    decimal_value = _binary_to_decimal128(binary_data, scale)
                     value_array.append(decimal_value)
         else:
             value_array.append(None)