e6data-python-connector 3.0.0__tar.gz → 3.0.1rc1__tar.gz

{e6data_python_connector-3.0.0/e6data_python_connector.egg-info → e6data_python_connector-3.0.1rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: e6data-python-connector
-Version: 3.0.0
+Version: 3.0.1rc1
 Summary: Client for the e6data distributed SQL Engine.
 Home-page: https://github.com/e6x-labs/e6data-python-connector
 Author: e6data, Inc.
@@ -469,6 +469,88 @@ database = '<new_database_name>'  # Replace with the new database.
 cursor = conn.cursor(database, catalog_name)
 ```
 
+### Opt in to multi-chunk result batches
+
+`enable_result_batch_v2=True` lets a compatible planner return several result
+chunks in one `getNextResultBatchV2` response. The default is `False`, which keeps
+the existing V1 protocol. Increasing `arraysize` or `fetchmany(size)` changes how
+many rows your application receives per call; it does not change the planner's
+response size.
+
+The planner must support V2 and have `ENABLE_GET_NEXT_RESULT_BATCH_V2` enabled.
+`ENABLE_GET_NEXT_CHUNK_V2` controls the separate executor-to-planner boundary.
+Record both flags when qualifying a deployment. V2 can reduce network round
+trips, but it does not reduce the number of result bytes or extend the planner's
+query lifetime. Completion of a 20-million-row result within 900 seconds must be
+measured against the actual workload and deployment.
+
+The following examples use your existing `connection_options`, `sql`, and
+`consume` function. Choose a positive `result_receive_limit_bytes` from measured
+V2 response sizes and the client's memory budget. The sync API preserves its
+existing gRPC options, so supply a finite receive limit for a V2 rollout:
+
+```python
+from e6data_python_connector import Connection
+
+sync_options = {
+    **connection_options,
+    "enable_result_batch_v2": True,
+    "grpc_options": {
+        **connection_options.get("grpc_options", {}),
+        "max_receive_message_length": result_receive_limit_bytes,
+    },
+}
+with Connection(**sync_options) as connection:
+    with connection.cursor() as cursor:
+        cursor.execute(sql)
+        for rows in cursor.fetchall_buffer():
+            consume(rows)
+```
+
+The async API keeps its finite 64 MiB default. Set
+`max_receive_message_bytes` to change that limit. If `grpc_options` also contains
+a receive limit, it must agree with this value.
+
+```python
+from e6data_python_connector.aio import AsyncConnection
+
+async_options = {
+    **connection_options,
+    "enable_result_batch_v2": True,
+    "max_receive_message_bytes": result_receive_limit_bytes,
+}
+async with AsyncConnection(**async_options) as connection:
+    async with connection.cursor() as cursor:
+        await cursor.execute(sql)
+        async for rows in cursor.fetchall_buffer():
+            consume(rows)
+```
+
+SQLAlchemy accepts the same opt-in through `connect_args`:
+
+```python
+from sqlalchemy import create_engine
+from sqlalchemy.ext.asyncio import create_async_engine
+
+engine = create_engine("e6data://", connect_args=sync_options)
+async_engine = create_async_engine("e6data+asyncio://", connect_args=async_options)
+```
+
+Fetch return shapes and row order stay the same. `fetchall_buffer()` yields
+decoded chunks; a V2 response can contain several chunks. All chunks in an
+envelope are decoded before any are exposed, so V2 may use more memory and delay
+the first chunk compared with V1. `fetchall()` still retains the full result.
+There is no prefetch or parallel fetching for a query.
+
+An `UNIMPLEMENTED` response switches that query to V1. A later query may try V2
+again. Other fetch failures do not trigger a protocol fallback or query replay.
+To disable V2 for new connections, omit the option or set it to `False`.
+
+The [result batch qualification instructions](test/README.md#result-batch-v2-qualification)
+cover protocol parity and the explicit large-result benchmark. Real-engine
+qualification and the 900-second acceptance measurement have not been run for
+this change.
+
 ### Get Query Time Metrics
 ```python
 import json

{e6data_python_connector-3.0.0 → e6data_python_connector-3.0.1rc1}/README.md

@@ -423,6 +423,88 @@ database = '<new_database_name>'  # Replace with the new database.
 cursor = conn.cursor(database, catalog_name)
 ```
 
+### Opt in to multi-chunk result batches
+
+`enable_result_batch_v2=True` lets a compatible planner return several result
+chunks in one `getNextResultBatchV2` response. The default is `False`, which keeps
+the existing V1 protocol. Increasing `arraysize` or `fetchmany(size)` changes how
+many rows your application receives per call; it does not change the planner's
+response size.
+
+The planner must support V2 and have `ENABLE_GET_NEXT_RESULT_BATCH_V2` enabled.
+`ENABLE_GET_NEXT_CHUNK_V2` controls the separate executor-to-planner boundary.
+Record both flags when qualifying a deployment. V2 can reduce network round
+trips, but it does not reduce the number of result bytes or extend the planner's
+query lifetime. Completion of a 20-million-row result within 900 seconds must be
+measured against the actual workload and deployment.
+
+The following examples use your existing `connection_options`, `sql`, and
+`consume` function. Choose a positive `result_receive_limit_bytes` from measured
+V2 response sizes and the client's memory budget. The sync API preserves its
+existing gRPC options, so supply a finite receive limit for a V2 rollout:
+
+```python
+from e6data_python_connector import Connection
+
+sync_options = {
+    **connection_options,
+    "enable_result_batch_v2": True,
+    "grpc_options": {
+        **connection_options.get("grpc_options", {}),
+        "max_receive_message_length": result_receive_limit_bytes,
+    },
+}
+with Connection(**sync_options) as connection:
+    with connection.cursor() as cursor:
+        cursor.execute(sql)
+        for rows in cursor.fetchall_buffer():
+            consume(rows)
+```
+
+The async API keeps its finite 64 MiB default. Set
+`max_receive_message_bytes` to change that limit. If `grpc_options` also contains
+a receive limit, it must agree with this value.
+
+```python
+from e6data_python_connector.aio import AsyncConnection
+
+async_options = {
+    **connection_options,
+    "enable_result_batch_v2": True,
+    "max_receive_message_bytes": result_receive_limit_bytes,
+}
+async with AsyncConnection(**async_options) as connection:
+    async with connection.cursor() as cursor:
+        await cursor.execute(sql)
+        async for rows in cursor.fetchall_buffer():
+            consume(rows)
+```
+
+SQLAlchemy accepts the same opt-in through `connect_args`:
+
+```python
+from sqlalchemy import create_engine
+from sqlalchemy.ext.asyncio import create_async_engine
+
+engine = create_engine("e6data://", connect_args=sync_options)
+async_engine = create_async_engine("e6data+asyncio://", connect_args=async_options)
+```
+
+Fetch return shapes and row order stay the same. `fetchall_buffer()` yields
+decoded chunks; a V2 response can contain several chunks. All chunks in an
+envelope are decoded before any are exposed, so V2 may use more memory and delay
+the first chunk compared with V1. `fetchall()` still retains the full result.
+There is no prefetch or parallel fetching for a query.
+
+An `UNIMPLEMENTED` response switches that query to V1. A later query may try V2
+again. Other fetch failures do not trigger a protocol fallback or query replay.
+To disable V2 for new connections, omit the option or set it to `False`.
+
+The [result batch qualification instructions](test/README.md#result-batch-v2-qualification)
+cover protocol parity and the explicit large-result benchmark. Real-engine
+qualification and the 900-second acceptance measurement have not been run for
+this change.
+
 ### Get Query Time Metrics
 ```python
 import json

{e6data_python_connector-3.0.0 → e6data_python_connector-3.0.1rc1}/e6data_python_connector/async_connection.py

@@ -50,7 +50,8 @@ class AsyncConnection:
                  token_url=None, oauth_scope=None, access_token=None,
                  client_auth_method='basic', *, operation_timeout=600.0,
                  oauth_timeout=10.0, cleanup_timeout=10.0, auto_resume_timeout=300.0,
-                 max_receive_message_bytes=64 * 1024 * 1024):
+                 max_receive_message_bytes=64 * 1024 * 1024,
+                 enable_result_batch_v2=False):
         if sys.version_info < (3, 11):
             raise RuntimeError('The async API requires Python 3.11 or newer.')
         try:
@@ -78,13 +79,16 @@ class AsyncConnection:
             raise ValueError('Async OAuth requires verified TLS (secure=True).')
         if ssl_cert is not None and not isinstance(ssl_cert, (str, bytes)):
             raise ValueError('ssl_cert must be a certificate path or PEM bytes.')
+        if not isinstance(enable_result_batch_v2, bool):
+            raise ValueError('enable_result_batch_v2 must be a boolean.')
         settings = dict(host=host, port=port, username=username, password=password,
                         catalog=catalog, database=database, cluster_name=cluster_name,
                         secure=bool(secure), ssl_cert=ssl_cert, auto_resume=auto_resume,
                         scheme=scheme, debug=debug, require_fastbinary=require_fastbinary,
                         client_id=client_id, client_secret=client_secret, token_url=token_url,
                         oauth_scope=oauth_scope, access_token=access_token,
-                        client_auth_method=client_auth_method)
+                        client_auth_method=client_auth_method,
+                        enable_result_batch_v2=enable_result_batch_v2)
         for name, value in [('operation_timeout', operation_timeout), ('oauth_timeout', oauth_timeout),
                             ('cleanup_timeout', cleanup_timeout), ('auto_resume_timeout', auto_resume_timeout)]:
             settings[name] = validate_positive_timeout(value, name)

{e6data_python_connector-3.0.0 → e6data_python_connector-3.0.1rc1}/e6data_python_connector/async_cursor.py

@@ -1,5 +1,7 @@
 """Native asynchronous query lifecycle, with no replay of consumed results."""
 import asyncio
+import logging
+import time
 from collections import deque
 from contextlib import contextmanager
 from io import BytesIO
@@ -11,6 +13,22 @@ from .e6data_grpc import _escaper
 from .exceptions import (AmbiguousSubmissionError, IncompleteResultError,
                          NotSupportedError, OperationalError, ProgrammingError, OAuthError)
 from .server import e6x_engine_pb2 as pb
+from .result_batch import ResultBatchBuffer, decode_result_batches
+
+
+logger = logging.getLogger(__name__)
+
+
+def _log_result_batch(protocol, rpc_seconds, status, response=None, decode_seconds=0.0):
+    logger.debug('Result batch fetch completed.', extra={
+        'result_batch_protocol': protocol,
+        'result_batch_rpc_seconds': rpc_seconds,
+        'result_batch_chunk_count': (len(response.resultBatches) if protocol == 'v2'
+                                     else int(bool(response.resultBatch))) if response else 0,
+        'result_batch_serialized_bytes': response.ByteSize() if response else 0,
+        'result_batch_decode_seconds': decode_seconds,
+        'result_batch_status': status,
+    })
 
 
 def _metadata_columns(payload):
@@ -46,8 +64,16 @@ class AsyncCursor:
         self._active_call = None
         self._close_task = None
         self._cleanup_error = None
+        self._result_batches = ResultBatchBuffer()
+        self._reset_result_protocol()
         connection._cursors.add(self)
 
+    def _reset_result_protocol(self):
+        self._result_batch_v2 = self._connection.enable_result_batch_v2
+        self._result_protocol = 'v2' if self._result_batch_v2 else 'v1'
+        self._result_session_id = None
+        self._result_batches.clear()
+
     @property
     def connection(self):
         return self._public_connection
@@ -137,6 +163,9 @@ class AsyncCursor:
             self._failure = IncompleteResultError(reason, query_id=self.query_id)
         if self._state != 'CLOSED':
             self._state = 'RESULT_FAILED'
+        if self._result_batch_v2:
+            self._rows.clear()
+            self._result_batches.clear()
         return self._failure
 
     def _accept_rows(self, rows):
@@ -147,10 +176,18 @@ class AsyncCursor:
         self._rownumber += len(rows)
         return rows
 
-    async def _query_request(self, cls, deadline):
+    async def _query_request(self, cls, deadline, *, owned=False):
         if self._route is None:
             raise ProgrammingError('No query handle is known.')
-        session = await self._connection.get_session_id(deadline=deadline)
+        if self._result_batch_v2 and not self._connection._oauth_enabled and self._result_session_id:
+            session = self._result_session_id
+        elif owned:
+            # Owned cleanup cannot mint a replacement authentication session.
+            session = '' if self._connection._oauth_enabled else self._connection._session_id
+            if session is None:
+                raise OperationalError('Cleanup has no authenticated legacy session.')
+        else:
+            session = await self._connection.get_session_id(deadline=deadline)
         return cls(sessionId=session, queryId=self.query_id, engineIP=self._route.engine_ip)
 
     async def _refresh_metadata(self, deadline):
@@ -168,6 +205,7 @@ class AsyncCursor:
         from .async_connection import QueryRoute
         if self._state != 'EMPTY':
             raise ProgrammingError('Clear the previous query before executing another.')
+        self._reset_result_protocol()
         if not isinstance(operation, str) or not operation.strip():
             raise ProgrammingError('operation must be a nonempty SQL string.')
         sql = operation.strip().removesuffix(';')
@@ -268,6 +306,8 @@ class AsyncCursor:
 
     async def _next_batch(self, deadline):
         from .async_work import reserve_work
+        if self._result_batch_v2:
+            return await self._next_v2_batch(deadline)
         if self._state == 'EXHAUSTED':
             return None
         if self._columns is None:
@@ -279,13 +319,23 @@ class AsyncCursor:
         def mark_dispatch():
             nonlocal consumed
             consumed = True
+        started = time.monotonic()
+        response = None
+        decode_seconds = 0.0
         try:
-            response = await self._call('getNextResultBatch', request, deadline, on_dispatch=mark_dispatch)
+            try:
+                response = await self._call('getNextResultBatch', request, deadline, on_dispatch=mark_dispatch)
+            finally:
+                rpc_seconds = time.monotonic() - started
             payload = response.resultBatch
             if not payload:
                 self._state = 'EXHAUSTED'
                 return None
-            return await self._decode_batch(payload, reservation, revision)
+            decode_started = time.monotonic()
+            try:
+                return await self._decode_batch(payload, reservation, revision)
+            finally:
+                decode_seconds = time.monotonic() - decode_started
         except asyncio.CancelledError:
             if consumed:
                 self._fail_result('ambiguous_result')
@@ -298,6 +348,110 @@ class AsyncCursor:
             raise
         finally:
             reservation.release()
+            if consumed:
+                status = 'ok' if response is not None and self._failure is None else 'error'
+                _log_result_batch('v1', rpc_seconds,
+                                  status, response if status == 'ok' else None, decode_seconds if status == 'ok' else 0.0)
+
+    def _retain_result_session(self, response, revision):
+        self._publish(revision)
+        if self._result_batch_v2 and not self._connection._oauth_enabled and response.sessionId:
+            self._result_session_id = response.sessionId
+
+    async def _decode_v2_response(self, response, reservation, revision):
+        self._retain_result_session(response, revision)
+        try:
+            chunks = await reservation.run(decode_result_batches, self._columns, response.resultBatches)
+        except Exception as error:
+            raise self._fail_result('decode_failed') from error
+        self._publish(revision)
+        self._result_batches.accept(chunks, response.endOfStream)
+
+    async def _next_v2_batch(self, deadline):
+        from .async_work import reserve_work
+        batch = self._result_batches.pop()
+        if batch is not None:
+            return batch
+        if self._result_batches.finished or self._state == 'EXHAUSTED':
+            self._state = 'EXHAUSTED'
+            return None
+        if self._columns is None:
+            await self._refresh_metadata(deadline)
+        consumed = False
+        backoff = .01
+        try:
+            while True:
+                request = await self._query_request(pb.GetNextResultBatchRequest, deadline)
+                reservation = await reserve_work(deadline=deadline)
+                revision = self._revision
+                protocol = self._result_protocol
+                previously_consumed = consumed
+                dispatched = False
+                response = None
+                status = 'error'
+                rpc_seconds = decode_seconds = 0.0
+                started = time.monotonic()
+
+                def mark_dispatch():
+                    nonlocal consumed, dispatched
+                    consumed = dispatched = True
+
+                try:
+                    try:
+                        response = await self._call(
+                            'getNextResultBatchV2' if protocol == 'v2' else 'getNextResultBatch',
+                            request, deadline, on_dispatch=mark_dispatch)
+                    except grpc.RpcError as error:
+                        if protocol == 'v2' and error.code() == grpc.StatusCode.UNIMPLEMENTED:
+                            consumed = previously_consumed
+                            self._result_protocol = 'v1'
+                            status = 'unimplemented'
+                            logger.debug('Result batch compatibility fallback.',
+                                         extra={'result_batch_fallback': True})
+                            continue
+                        raise
+                    finally:
+                        rpc_seconds = time.monotonic() - started
+                    decode_started = time.monotonic()
+                    try:
+                        if protocol == 'v2':
+                            await self._decode_v2_response(response, reservation, revision)
+                        else:
+                            self._retain_result_session(response, revision)
+                            if response.resultBatch:
+                                batch = await self._decode_batch(response.resultBatch, reservation, revision)
+                            else:
+                                self._state = 'EXHAUSTED'
+                                batch = None
+                    finally:
+                        decode_seconds = time.monotonic() - decode_started
+                    status = 'ok'
+                    if protocol == 'v1':
+                        return batch
+                    batch = self._result_batches.pop()
+                    if batch is not None:
+                        return batch
+                    if self._result_batches.finished:
+                        self._state = 'EXHAUSTED'
+                        return None
+                finally:
+                    reservation.release()
+                    if dispatched:
+                        _log_result_batch(protocol, rpc_seconds, status,
+                                          response if status == 'ok' else None,
+                                          decode_seconds if status == 'ok' else 0.0)
+                await asyncio.sleep(min(backoff, self._connection._remaining(deadline)))
+                backoff = min(backoff * 2, .1)
+        except asyncio.CancelledError:
+            if consumed:
+                self._fail_result('ambiguous_result')
+            raise
+        except IncompleteResultError:
+            raise
+        except Exception as error:
+            if consumed:
+                raise self._fail_result('ambiguous_result') from error
+            raise
 
     async def _fetchmany(self, size, deadline):
         while len(self._rows) < size and self._state != 'EXHAUSTED':
@@ -451,27 +605,25 @@ class AsyncCursor:
         if self._route is None:
             if self._state == 'SUBMISSION_UNKNOWN' or self in self._connection._ambiguous_submissions:
                 raise AmbiguousSubmissionError()
+            self._reset_query_state()
             return None
-        if owned:
-            # Owned cleanup has no authority to mint a new legacy session silently.
-            session = '' if self._connection._oauth_enabled else self._connection._session_id
-            if session is None:
-                raise OperationalError('Cleanup has no authenticated legacy session.')
-        else:
-            session = await self._connection.get_session_id(deadline=deadline)
-        request = pb.ClearOrCancelQueryRequest(sessionId=session, queryId=self.query_id, engineIP=self._route.engine_ip)
+        request = await self._query_request(pb.ClearOrCancelQueryRequest, deadline, owned=owned)
         response = await self._connection._rpc('clearOrCancelQuery', request, deadline=deadline,
                                               route=self._route, _cleanup=owned)
         self._connection._release_route(self.query_id)
         self._connection._ambiguous_submissions.discard(self)
+        self._reset_query_state()
+        return response
+
+    def _reset_query_state(self):
         self._route = None
         self._rows.clear()
+        self._reset_result_protocol()
         self._columns = self._description = None
         self._rowcount, self._rownumber = -1, 0
         self._failure = self._cleanup_error = None
         if self._state != 'CLOSED':
             self._state = 'EMPTY'
-        return response
 
     async def clear(self, query_id=None, *, timeout=None):
         self._check()
@@ -496,6 +648,8 @@ class AsyncCursor:
         try:
             async with asyncio.timeout_at(deadline):
                 await self._stop_active()
+                if self._result_batch_v2:
+                    self._fail_result('cancelled_result')
                 request = await self._query_request(pb.CancelQueryRequest, deadline)
                 await self._connection._rpc('cancelQuery', request, deadline=deadline, route=self._route)
                 self._fail_result('cancelled_result')
@@ -524,6 +678,8 @@ class AsyncCursor:
             self._cleanup_error = OperationalError('Query cleanup is unconfirmed.')
         finally:
             self._rows.clear()
+            self._result_batches.clear()
+            self._result_session_id = None
             self._connection._cursors.discard(self)
 
     async def close(self):

{e6data_python_connector-3.0.0 → e6data_python_connector-3.0.1rc1}/e6data_python_connector/async_dialect.py

@@ -216,7 +216,8 @@ class E6dataAsyncDialect(E6dataDialect):
         aliases = {'schema': 'database', 'cluster-name': 'cluster_name',
                    'cluster-uuid': 'cluster_name', 'auto-resume': 'auto_resume'}
         strings = {'catalog', 'database', 'cluster_name', 'ssl_cert'}
-        booleans = {'secure', 'auto_resume', 'debug', 'require_fastbinary'}
+        booleans = {'secure', 'auto_resume', 'debug', 'require_fastbinary',
+                    'enable_result_batch_v2'}
         floats = {'operation_timeout', 'cleanup_timeout', 'oauth_timeout', 'auto_resume_timeout'}
         options = {}
         if url.host is not None:

{e6data_python_connector-3.0.0 → e6data_python_connector-3.0.1rc1}/e6data_python_connector/dialect.py

@@ -248,6 +248,9 @@ class E6dataDialect(default.DefaultDialect):
         self.secure = url.query.get("secure") == "true"
         self.auto_resume = url.query.get("auto-resume", "true") == "true"  # default to True
         self.debug = url.query.get("debug", "false") == "true"  # default to True
+        result_batch_v2 = url.query.get('enable_result_batch_v2', 'false')
+        if result_batch_v2 not in ('true', 'false'):
+            raise ValueError('enable_result_batch_v2 must be true or false.')
         if not self.catalog_name:
             raise Exception('Please specify catalog in query parameter.')
 
@@ -255,7 +258,7 @@ class E6dataDialect(default.DefaultDialect):
         grpc_options = {}
         for key, value in url.query.items():
             # Skip known parameters that are not grpc_options
-            if key not in {"schema", "catalog", "cluster-uuid", "secure", "auto-resume"}:
+            if key not in {"schema", "catalog", "cluster-uuid", "secure", "auto-resume", "enable_result_batch_v2"}:
                 grpc_options[key] = value
 
         kwargs = {
@@ -270,8 +273,10 @@ class E6dataDialect(default.DefaultDialect):
             'secure': self.secure,
             'auto_resume': self.auto_resume,
             'grpc_options': grpc_options,
-            'debug': self.debug
+            'debug': self.debug,
         }
+        if 'enable_result_batch_v2' in url.query:
+            kwargs['enable_result_batch_v2'] = result_batch_v2 == 'true'
         return [], kwargs
 
     def get_schema_names(self, connection, **kw):