e6data-python-connector 3.0.1rc2__tar.gz → 3.0.1rc4__tar.gz

{e6data_python_connector-3.0.1rc2/e6data_python_connector.egg-info → e6data_python_connector-3.0.1rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e6data-python-connector
-Version: 3.0.1rc2
+Version: 3.0.1rc4
 Summary: Client for the e6data distributed SQL Engine.
 Home-page: https://github.com/e6x-labs/e6data-python-connector
 Author: e6data, Inc.
@@ -489,11 +489,15 @@ workers use the explicit `spawn` method; the application's global multiprocessin
 start method is unchanged. An unguarded main script can run its other import
 side effects, so keep all application startup inside the guard.
 
-Both APIs default to a 64 MiB receive limit when V2 is enabled. A positive finite
-custom limit is allowed; unlimited receive sizes are rejected for V2. The sync
-flag-off path keeps its existing options. For async, set
-`max_receive_message_bytes`; any receive limit in `grpc_options` must agree.
-Choose limits from measured envelope sizes and the client's memory budget.
+Both APIs default to unlimited gRPC send and receive message sizes (`-1`).
+Explicit positive byte limits are still honored. For sync, set
+`max_receive_message_length` and `max_send_message_length` in `grpc_options`.
+For async, set the receive limit with `max_receive_message_bytes` or
+`max_receive_message_length` in `grpc_options`; if both are supplied, they must
+agree. Set `max_send_message_length` in `grpc_options` for the send limit.
+The receive limit applies to each message after transport decompression, not the
+total query result. Unlimited receiving does not limit client memory use;
+continuous downloading can retain serialized batches while decoding catches up.
 
 These examples use your existing `connection_options`, `sql`, and `consume`:
 
@@ -567,6 +571,15 @@ downloading is faster than decoding or application reads. Workers add Python
 objects and process copies; the receive limit is not a limit on total memory.
 `fetchall()` still retains the full result.
 
+If a decoder worker exits or its local pipe breaks, the connector stops the
+failed workers and decodes that complete serialized envelope once in the parent
+process. It uses the same deadline and makes no extra result RPC. Continuous
+downloading stays active and uncapped during this recovery. Later envelopes use
+sequential decoding until the decoder runtime is replaced through its normal
+lifecycle. Cancellation, expired deadlines, invalid worker replies, and errors
+reported while decoding data are still failures. Their original cause is kept
+on the first error delivered to the caller.
+
 Only a V2 `UNIMPLEMENTED` response switches that query to V1 and disables its
 optimizations. Other RPC failures reach the caller as the original
 `grpc.RpcError` or `grpc.aio.AioRpcError`. A failed pending RPC is raised when

{e6data_python_connector-3.0.1rc2 → e6data_python_connector-3.0.1rc4}/README.md

@@ -443,11 +443,15 @@ workers use the explicit `spawn` method; the application's global multiprocessin
 start method is unchanged. An unguarded main script can run its other import
 side effects, so keep all application startup inside the guard.
 
-Both APIs default to a 64 MiB receive limit when V2 is enabled. A positive finite
-custom limit is allowed; unlimited receive sizes are rejected for V2. The sync
-flag-off path keeps its existing options. For async, set
-`max_receive_message_bytes`; any receive limit in `grpc_options` must agree.
-Choose limits from measured envelope sizes and the client's memory budget.
+Both APIs default to unlimited gRPC send and receive message sizes (`-1`).
+Explicit positive byte limits are still honored. For sync, set
+`max_receive_message_length` and `max_send_message_length` in `grpc_options`.
+For async, set the receive limit with `max_receive_message_bytes` or
+`max_receive_message_length` in `grpc_options`; if both are supplied, they must
+agree. Set `max_send_message_length` in `grpc_options` for the send limit.
+The receive limit applies to each message after transport decompression, not the
+total query result. Unlimited receiving does not limit client memory use;
+continuous downloading can retain serialized batches while decoding catches up.
 
 These examples use your existing `connection_options`, `sql`, and `consume`:
 
@@ -521,6 +525,15 @@ downloading is faster than decoding or application reads. Workers add Python
 objects and process copies; the receive limit is not a limit on total memory.
 `fetchall()` still retains the full result.
 
+If a decoder worker exits or its local pipe breaks, the connector stops the
+failed workers and decodes that complete serialized envelope once in the parent
+process. It uses the same deadline and makes no extra result RPC. Continuous
+downloading stays active and uncapped during this recovery. Later envelopes use
+sequential decoding until the decoder runtime is replaced through its normal
+lifecycle. Cancellation, expired deadlines, invalid worker replies, and errors
+reported while decoding data are still failures. Their original cause is kept
+on the first error delivered to the caller.
+
 Only a V2 `UNIMPLEMENTED` response switches that query to V1 and disables its
 optimizations. Other RPC failures reach the caller as the original
 `grpc.RpcError` or `grpc.aio.AioRpcError`. A failed pending RPC is raised when

{e6data_python_connector-3.0.1rc2 → e6data_python_connector-3.0.1rc4}/e6data_python_connector/async_cluster_manager.py

@@ -123,10 +123,15 @@ class AsyncClusterManager:
         options = dict(grpc_options or {})
         if any(key.removeprefix('grpc.') in ('ssl_target_name_override', 'default_authority') for key in options):
             raise ValueError('TLS authority overrides are unsupported.')
-        limit = options.get('grpc.max_receive_message_length', 64 * 1024 * 1024)
-        if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
-            raise ValueError('A finite positive receive limit is required.')
-        options['grpc.max_receive_message_length'] = limit
+        for name in ('max_receive_message_length', 'max_send_message_length'):
+            limits = [options.pop(key) for key in (name, 'grpc.' + name) if key in options]
+            for limit in limits:
+                if (isinstance(limit, bool) or not isinstance(limit, int)
+                        or (limit != -1 and limit <= 0)):
+                    raise ValueError(name + ' must be -1 (unlimited) or a positive integer.')
+            if len(limits) == 2 and limits[0] != limits[1]:
+                raise ValueError('Conflicting ' + name + ' options.')
+            options['grpc.' + name] = limits[0] if limits else -1
         self._host, self._port = host, port
         self._user, self._password = user, password
         self._secure_channel, self._ssl_cert = secure_channel, ssl_cert

{e6data_python_connector-3.0.1rc2 → e6data_python_connector-3.0.1rc4}/e6data_python_connector/async_connection.py

@@ -14,6 +14,9 @@ from .oauth_common import validate_positive_timeout, validate_token_endpoint
 from .server import e6x_engine_pb2 as pb, e6x_engine_pb2_grpc as bindings
 
 
+_UNSET_RECEIVE_MESSAGE_BYTES = object()
+
+
 @dataclass(frozen=True)
 class QueryRoute:
     target: str
@@ -50,7 +53,7 @@ class AsyncConnection:
                  token_url=None, oauth_scope=None, access_token=None,
                  client_auth_method='basic', *, operation_timeout=600.0,
                  oauth_timeout=10.0, cleanup_timeout=10.0, auto_resume_timeout=300.0,
-                 max_receive_message_bytes=64 * 1024 * 1024,
+                 max_receive_message_bytes=_UNSET_RECEIVE_MESSAGE_BYTES,
                  enable_result_batch_v2=False):
         from .result_decode_worker import is_decode_worker
         if is_decode_worker():
@@ -98,14 +101,31 @@ class AsyncConnection:
         for name, value in [('operation_timeout', operation_timeout), ('oauth_timeout', oauth_timeout),
                             ('cleanup_timeout', cleanup_timeout), ('auto_resume_timeout', auto_resume_timeout)]:
             settings[name] = validate_positive_timeout(value, name)
-        if isinstance(max_receive_message_bytes, bool) or not isinstance(max_receive_message_bytes, int) or max_receive_message_bytes <= 0:
-            raise ValueError('max_receive_message_bytes must be a positive integer.')
         options = dict(grpc_options or {})
-        for name in ('max_receive_message_length', 'grpc.max_receive_message_length'):
-            if name in options:
-                if options[name] != max_receive_message_bytes:
-                    raise ValueError('Set the finite receive limit using max_receive_message_bytes.')
-                del options[name]
+        receive_names = ('max_receive_message_length', 'grpc.max_receive_message_length')
+        receive_values = [options.pop(name) for name in receive_names if name in options]
+        for value in receive_values:
+            if (isinstance(value, bool) or not isinstance(value, int)
+                    or (value != -1 and value <= 0)):
+                raise ValueError('max_receive_message_length must be -1 (unlimited) or a positive integer.')
+        if len(receive_values) == 2 and receive_values[0] != receive_values[1]:
+            raise ValueError('Conflicting max_receive_message_length options.')
+        if max_receive_message_bytes is _UNSET_RECEIVE_MESSAGE_BYTES:
+            max_receive_message_bytes = receive_values[0] if receive_values else -1
+        if (isinstance(max_receive_message_bytes, bool) or not isinstance(max_receive_message_bytes, int)
+                or (max_receive_message_bytes != -1 and max_receive_message_bytes <= 0)):
+            raise ValueError('max_receive_message_bytes must be -1 (unlimited) or a positive integer.')
+        if receive_values and receive_values[0] != max_receive_message_bytes:
+            raise ValueError('Set the receive limit using max_receive_message_bytes; supplied options must agree.')
+        send_names = ('max_send_message_length', 'grpc.max_send_message_length')
+        send_values = [options.pop(name) for name in send_names if name in options]
+        for value in send_values:
+            if (isinstance(value, bool) or not isinstance(value, int)
+                    or (value != -1 and value <= 0)):
+                raise ValueError('max_send_message_length must be -1 (unlimited) or a positive integer.')
+        if len(send_values) == 2 and send_values[0] != send_values[1]:
+            raise ValueError('Conflicting max_send_message_length options.')
+        options['grpc.max_send_message_length'] = send_values[0] if send_values else -1
         # Authority overrides can disable meaningful server identity validation.
         if any(name.removeprefix('grpc.') in ('ssl_target_name_override', 'default_authority') for name in options):
             raise ValueError('TLS authority overrides are unsupported.')
@@ -385,6 +405,8 @@ class AsyncConnection:
 
             self._cluster_manager = AsyncClusterManager(
                 self.host, self.port, user=self.username or '', password=self.password or '',
+                grpc_options={**self.grpc_options,
+                              'grpc.max_receive_message_length': self.max_receive_message_bytes},
                 secure_channel=self.secure, cluster_uuid=self.cluster_name,
                 ssl_cert=self.ssl_cert, metadata_provider=metadata_provider if self._oauth_enabled else None,
                 initial_strategy=self.strategy, auto_resume_timeout=self.auto_resume_timeout,

{e6data_python_connector-3.0.1rc2 → e6data_python_connector-3.0.1rc4}/e6data_python_connector/async_connection_pool.py

@@ -189,6 +189,8 @@ class AsyncConnectionPool:
             from .async_cluster_manager import AsyncClusterManager
             self._cluster_manager = AsyncClusterManager(
                 config.host, config.port, user=config.username or '', password=config.password or '',
+                grpc_options={**config.grpc_options,
+                              'grpc.max_receive_message_length': config.max_receive_message_bytes},
                 secure_channel=config.secure, ssl_cert=config.ssl_cert,
                 cluster_uuid=config.cluster_name, initial_strategy='blue',
                 metadata_provider=self._resume_metadata if config._oauth_enabled else None,

{e6data_python_connector-3.0.1rc2 → e6data_python_connector-3.0.1rc4}/e6data_python_connector/async_cursor.py

@@ -129,7 +129,7 @@ class AsyncCursor:
         if self._state == 'CLOSED':
             raise ProgrammingError('Cursor is closed.')
         if fetch and self._failure is not None:
-            raise self._failure
+            raise IncompleteResultError(self._failure.reason, query_id=self._failure.query_id)
         if fetch and self._state in ('EMPTY', 'SUBMISSION_UNKNOWN'):
             raise ProgrammingError('No complete query is available for fetching.')
 
@@ -172,12 +172,13 @@ class AsyncCursor:
         self._retire_prefetch()
         self._cancel_decode()
         if self._failure is None:
+            # Never raise this template: its traceback would retain decoded inputs.
             self._failure = IncompleteResultError(reason, query_id=self.query_id)
         if self._state != 'CLOSED':
             self._state = 'RESULT_FAILED'
         self._rows.clear()
         self._result_batches.clear()
-        return self._failure
+        return IncompleteResultError(self._failure.reason, query_id=self._failure.query_id)
 
     def _accept_rows(self, rows):
         self._rows.extend(rows)

{e6data_python_connector-3.0.1rc2 → e6data_python_connector-3.0.1rc4}/e6data_python_connector/cluster_manager.py

@@ -1,4 +1,6 @@
+import hashlib
 import logging
+import os
 import threading
 import time
 import e6data_python_connector.cluster_server.cluster_pb2 as cluster_pb2
@@ -74,6 +76,7 @@ class _StatusLock:
         self._status_multiprocessing_lock = (threading.Semaphore() if is_decode_worker()
                                              else multiprocessing.Semaphore())
         self._is_active = False
+        self._resume_completion = None
 
     @property
     def is_active(self) -> bool:
@@ -109,8 +112,14 @@ class _StatusLock:
            _StatusLock: The current instance of the lock, used for context management.
         """
 
-        self._status_thread_lock.acquire(timeout=self._LOCK_TIMEOUT)
-        self._status_multiprocessing_lock.acquire(timeout=self._LOCK_TIMEOUT)
+        if not self._status_thread_lock.acquire(timeout=self._LOCK_TIMEOUT):
+            raise TimeoutError('Cluster recovery timed out waiting for status lock.')
+        try:
+            if not self._status_multiprocessing_lock.acquire(timeout=self._LOCK_TIMEOUT):
+                raise TimeoutError('Cluster recovery timed out waiting for process lock.')
+        except BaseException:
+            self._status_thread_lock.release()
+            raise
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -127,8 +136,8 @@ class _StatusLock:
             exc_tb (Traceback): The traceback object of the exception (if any).
         """
 
-        self._status_thread_lock.release()
         self._status_multiprocessing_lock.release()
+        self._status_thread_lock.release()
 
     @contextmanager
     def hold_until(self, deadline):
@@ -196,9 +205,10 @@ class ClusterManager:
         self._timeout = time.time() + timeout
         self._secure_channel = secure_channel
         self.cluster_uuid = cluster_uuid
-        self._grpc_options = grpc_options
-        if grpc_options is None:
-            self._grpc_options = dict()
+        options = dict(grpc_options or ())
+        options.setdefault('grpc.max_receive_message_length', -1)
+        options.setdefault('grpc.max_send_message_length', -1)
+        self._grpc_options = list(options.items())
         self._debug = debug
         self._ssl_cert = ssl_cert
         self._metadata_provider = metadata_provider
@@ -465,6 +475,34 @@ class ClusterManager:
                     logger.warning(f"Status check failed with error: {e}")
                 yield None
 
+    def _legacy_resume_identity(self):
+        """Fingerprint exact connection inputs without retaining credentials.
+
+        Copy standard option containers. Unknown or mutable option values disable
+        sharing so their existing RPC behavior is preserved without guessing.
+        """
+        values = (self._host, self._port, self.cluster_uuid, self._secure_channel,
+                  self._ssl_cert, self._user, self._password)
+        scalar_types = (type(None), str, bytes, bool, int)
+        if any(type(value) not in scalar_types for value in values):
+            return None
+        options = self._grpc_options
+        if type(options) is dict:
+            pairs = tuple(options.items())
+        elif type(options) in (list, tuple):
+            pairs = tuple(options)
+        else:
+            return None
+        frozen_options = []
+        for pair in pairs:
+            if (type(pair) not in (tuple, list) or len(pair) != 2
+                    or any(type(value) not in scalar_types for value in pair)):
+                return None
+            frozen_options.append(tuple((type(value).__name__, value) for value in pair))
+        identity = (tuple((type(value).__name__, value) for value in values),
+                    type(options).__name__, tuple(frozen_options))
+        return hashlib.sha256(repr(identity).encode('utf-8')).digest()
+
     def resume(self) -> bool:
         """
         Resumes the cluster if it is currently suspended or not in the 'active' state.
@@ -494,14 +532,21 @@ class ClusterManager:
         """
         if self._metadata_provider is not None:
             return self._resume_oauth()
+        call_started = time.monotonic()
         if self._debug:
             logger.info(f"Starting auto-resume for cluster {self.cluster_uuid} at {self._host}:{self._port}")
 
         with status_lock as lock:
-            if lock.is_active:
-                if self._debug:
-                    logger.info("Lock is already active, cluster appears to be running")
+            identity = self._legacy_resume_identity()
+            process_id = os.getpid()
+            completion = lock._resume_completion
+            if (identity is not None and completion is not None
+                    and completion[0] == process_id and completion[1] == identity
+                    and completion[2] > call_started):
                 return True
+            # A fresh attempt can observe a new failure before older waiters run.
+            # Invalidate first, including when the attempt later raises an error.
+            lock._resume_completion = None
 
             # Retrieve the current cluster status with strategy header
             if self._debug:
@@ -530,6 +575,8 @@ class ClusterManager:
             elif current_status.status == 'active':
                 if self._debug:
                     logger.info("Cluster is already active, no action needed")
+                if identity is not None:
+                    lock._resume_completion = (process_id, identity, time.monotonic())
                 return True
             elif current_status.status != 'resuming':
                 """
@@ -548,6 +595,8 @@ class ClusterManager:
                 if status == 'active':
                     if self._debug:
                         logger.info(f"Cluster became active after {check_count} status checks")
+                    if identity is not None:
+                        lock._resume_completion = (process_id, identity, time.monotonic())
                     return True
                 elif status == 'failed':
                     if self._debug:

{e6data_python_connector-3.0.1rc2 → e6data_python_connector-3.0.1rc4}/e6data_python_connector/e6data_grpc.py

@@ -527,9 +527,9 @@ class Connection(object):
                 if key in normalized and normalized[key] != value:
                     raise ValueError('Conflicting gRPC options after prefix normalization.')
                 normalized[key] = value
-            limit = normalized.get('max_receive_message_length', 64 * 1024 * 1024)
-            if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
-                raise ValueError('V2 receive limit must be a finite positive integer.')
+            limit = normalized.get('max_receive_message_length', -1)
+            if isinstance(limit, bool) or not isinstance(limit, int) or (limit != -1 and limit <= 0):
+                raise ValueError('V2 receive limit must be a positive integer or -1 for unlimited.')
             normalized['max_receive_message_length'] = limit
             grpc_options = normalized
         if enable_result_batch_v2 and grpc_options is not None and 'grpc_prepare_timeout' in grpc_options:
@@ -696,7 +696,7 @@ class Connection(object):
             default_options = {
                 "keepalive_timeout_ms": 900000,  # Time in milliseconds to keep the connection alive.
                 "max_receive_message_length": -1,  # Maximum size of received messages.
-                "max_send_message_length": 300 * 1024 * 1024,  # Maximum size of sent messages (300 MB).
+                "max_send_message_length": -1,  # Unlimited sent message size.
                 "grpc_prepare_timeout": self.grpc_prepare_timeout,  # Timeout for prepare statement API call.
                 "keepalive_time_ms": 30000,  # Time in milliseconds between keep-alive pings.
                 "keepalive_permit_without_calls": 1,  # Allow keep-alives with no active RPCs.
@@ -1032,6 +1032,7 @@ class Connection(object):
                     cluster_uuid=self.cluster_name,
                     timeout=self.grpc_auto_resume_timeout_seconds,
                     debug=self._debug,
+                    grpc_options=self._get_grpc_options,
                     ssl_cert=self._ssl_cert
                 ).resume()
                 return status  # Return boolean status directly
@@ -1709,7 +1710,9 @@ class Cursor(DBAPICursor):
                 self._result_failure = IncompleteResultError(reason, query_id=self._query_id)
             self._data = None
             self._result_batches.clear()
-            return self._result_failure
+            # Keep the cached template free of tracebacks and response frames.
+            return IncompleteResultError(
+                self._result_failure.reason, query_id=self._result_failure.query_id)
 
     def _check_result(self):
         if self._result_batch_v2_enabled and (self._result_pid != os.getpid() or
@@ -1720,7 +1723,8 @@ class Cursor(DBAPICursor):
         if self._closed:
             raise ProgrammingError('Cursor is closed.')
         if self._result_failure is not None:
-            raise self._result_failure
+            raise IncompleteResultError(
+                self._result_failure.reason, query_id=self._result_failure.query_id)
 
     @property
     def query_id(self):