e6data-python-connector 2.3.10rc6__tar.gz → 2.3.10rc8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {e6data_python_connector-2.3.10rc6/e6data_python_connector.egg-info → e6data_python_connector-2.3.10rc8}/PKG-INFO +10 -203
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/README.md +2 -195
- e6data_python_connector-2.3.10rc8/e6data_python_connector/__init__.py +3 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/e6data_grpc.py +1 -1
- e6data_python_connector-2.3.10rc8/e6data_python_connector/server/e6x_engine_pb2.py +207 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/server/e6x_engine_pb2.pyi +560 -276
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/server/e6x_engine_pb2_grpc.py +807 -282
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8/e6data_python_connector.egg-info}/PKG-INFO +10 -203
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector.egg-info/SOURCES.txt +0 -6
- e6data_python_connector-2.3.10rc8/e6data_python_connector.egg-info/requires.txt +8 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/setup.py +8 -8
- e6data_python_connector-2.3.10rc6/e6data_python_connector/__init__.py +0 -4
- e6data_python_connector-2.3.10rc6/e6data_python_connector/connection_pool.py +0 -495
- e6data_python_connector-2.3.10rc6/e6data_python_connector/server/e6x_engine_pb2.py +0 -141
- e6data_python_connector-2.3.10rc6/e6data_python_connector.egg-info/requires.txt +0 -8
- e6data_python_connector-2.3.10rc6/test/test_connection_pool.py +0 -410
- e6data_python_connector-2.3.10rc6/test/test_connection_pool_e2e.py +0 -564
- e6data_python_connector-2.3.10rc6/test/test_pool_concurrency_simple.py +0 -573
- e6data_python_connector-2.3.10rc6/test/test_pool_threading_multiprocessing.py +0 -679
- e6data_python_connector-2.3.10rc6/test/test_simple_connection_pool.py +0 -278
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/LICENSE +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/MANIFEST.in +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/cluster_manager.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/cluster_server/__init__.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/cluster_server/cluster_pb2.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/cluster_server/cluster_pb2_grpc.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/common.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/constants.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/datainputstream.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/date_time_utils.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/dialect.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/e6x_vector/constants.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/e6x_vector/ttypes.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/exceptions.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/server/QueryEngineService.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/server/__init__.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/server/constants.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/server/ttypes.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/strategy.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector/typeId.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector.egg-info/dependency_links.txt +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector.egg-info/entry_points.txt +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/e6data_python_connector.egg-info/top_level.txt +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/pyproject.toml +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/setup.cfg +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/__init__.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/analyze_38_nines.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/analyze_all_cases.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/analyze_binary.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/analyze_correct_value.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/analyze_fields.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/check_decimal_errors.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/cleanup_test_files.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/debug_38_nines.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/debug_binary.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/final_test.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/move_tests.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/quick_test.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_38_nines.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_all_decimal128_cases.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_cluster_manager_efficiency.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_cluster_manager_none_strategy.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_cluster_manager_strategy.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_comprehensive.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_current_implementation.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_decimal128_binary_parsing.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_decimal128_parsing.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_fix.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_improved_parsing.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_known_case.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_manual_analysis.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_mock_server.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_multiprocessing_fix.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_new_implementation.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_specific_binary.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_strategy.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_strategy_logic.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_strategy_persistence_fix.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_strategy_sharing_fix.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/test_user_binary_value.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/tests.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/tests_grpc.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/validate_decimal128.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/validate_implementation.py +0 -0
- {e6data_python_connector-2.3.10rc6 → e6data_python_connector-2.3.10rc8}/test/verify_decimal_fix.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.10rc8
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
6
|
Author: e6data, Inc.
|
|
@@ -19,13 +19,13 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE
|
|
21
21
|
Requires-Dist: sqlalchemy>=1.0.0
|
|
22
|
-
Requires-Dist: future
|
|
23
|
-
Requires-Dist: python-dateutil
|
|
24
|
-
Requires-Dist: pycryptodome
|
|
25
|
-
Requires-Dist: pytz
|
|
26
|
-
Requires-Dist: thrift
|
|
27
|
-
Requires-Dist: grpcio
|
|
28
|
-
Requires-Dist: grpcio-tools
|
|
22
|
+
Requires-Dist: future==1.0.0
|
|
23
|
+
Requires-Dist: python-dateutil==2.9.0.post0
|
|
24
|
+
Requires-Dist: pycryptodome==3.19.1
|
|
25
|
+
Requires-Dist: pytz==2024.1
|
|
26
|
+
Requires-Dist: thrift==0.20.0
|
|
27
|
+
Requires-Dist: grpcio==1.65.1
|
|
28
|
+
Requires-Dist: grpcio-tools
|
|
29
29
|
Dynamic: author
|
|
30
30
|
Dynamic: author-email
|
|
31
31
|
Dynamic: classifier
|
|
@@ -38,7 +38,7 @@ Dynamic: summary
|
|
|
38
38
|
|
|
39
39
|
# e6data Python Connector
|
|
40
40
|
|
|
41
|
-

|
|
42
42
|
|
|
43
43
|
## Introduction
|
|
44
44
|
|
|
@@ -75,8 +75,6 @@ Use your e6data Email ID as the username and your access token as the password.
|
|
|
75
75
|
|
|
76
76
|
```python
|
|
77
77
|
from e6data_python_connector import Connection
|
|
78
|
-
# For connection pooling (recommended for concurrent operations)
|
|
79
|
-
from e6data_python_connector import ConnectionPool
|
|
80
78
|
|
|
81
79
|
username = '<username>' # Your e6data Email ID.
|
|
82
80
|
password = '<password>' # Access Token generated in the e6data console.
|
|
@@ -86,7 +84,6 @@ database = '<database>' # Database to perform the query on.
|
|
|
86
84
|
port = 80 # Port of the e6data engine.
|
|
87
85
|
catalog_name = '<catalog_name>'
|
|
88
86
|
|
|
89
|
-
# Single connection (for simple, single-threaded use)
|
|
90
87
|
conn = Connection(
|
|
91
88
|
host=host,
|
|
92
89
|
port=port,
|
|
@@ -94,17 +91,6 @@ conn = Connection(
|
|
|
94
91
|
database=database,
|
|
95
92
|
password=password
|
|
96
93
|
)
|
|
97
|
-
|
|
98
|
-
# Or use connection pool (for concurrent/multi-threaded use)
|
|
99
|
-
pool = ConnectionPool(
|
|
100
|
-
min_size=2,
|
|
101
|
-
max_size=10,
|
|
102
|
-
host=host,
|
|
103
|
-
port=port,
|
|
104
|
-
username=username,
|
|
105
|
-
database=database,
|
|
106
|
-
password=password
|
|
107
|
-
)
|
|
108
94
|
```
|
|
109
95
|
|
|
110
96
|
#### Connection Parameters
|
|
@@ -453,187 +439,8 @@ For detailed migration instructions, see the [Migration Guide](docs/zero-downtim
|
|
|
453
439
|
- Message size optimization for large queries
|
|
454
440
|
|
|
455
441
|
### Connection Management
|
|
456
|
-
|
|
457
|
-
#### Connection Pooling
|
|
458
|
-
|
|
459
|
-
The e6data Python connector now includes a built-in connection pool for efficient connection management and reuse across multiple threads. The `ConnectionPool` class provides:
|
|
460
|
-
|
|
461
|
-
- **Thread-safe connection reuse**: Each thread automatically reuses its assigned connection
|
|
462
|
-
- **Automatic lifecycle management**: Handles connection creation, health checks, and cleanup
|
|
463
|
-
- **Overflow connections**: Creates temporary connections when pool is exhausted
|
|
464
|
-
- **Connection health monitoring**: Automatic detection and replacement of broken connections
|
|
465
|
-
- **Statistics tracking**: Monitor pool usage and performance
|
|
466
|
-
|
|
467
|
-
##### Basic Connection Pool Usage
|
|
468
|
-
|
|
469
|
-
```python
|
|
470
|
-
from e6data_python_connector import ConnectionPool
|
|
471
|
-
|
|
472
|
-
# Create a connection pool
|
|
473
|
-
pool = ConnectionPool(
|
|
474
|
-
min_size=2, # Minimum connections to maintain
|
|
475
|
-
max_size=10, # Maximum connections in pool
|
|
476
|
-
max_overflow=5, # Additional temporary connections allowed
|
|
477
|
-
timeout=30.0, # Timeout for getting connection (seconds)
|
|
478
|
-
recycle=3600, # Maximum age before recycling (seconds)
|
|
479
|
-
debug=False, # Enable debug logging
|
|
480
|
-
pre_ping=True, # Check connection health before use
|
|
481
|
-
# Connection parameters
|
|
482
|
-
host=host,
|
|
483
|
-
port=port,
|
|
484
|
-
username=username,
|
|
485
|
-
password=password,
|
|
486
|
-
database=database,
|
|
487
|
-
catalog=catalog_name,
|
|
488
|
-
cluster_name=cluster_name,
|
|
489
|
-
secure=True
|
|
490
|
-
)
|
|
491
|
-
|
|
492
|
-
# Get connection and execute query
|
|
493
|
-
conn = pool.get_connection()
|
|
494
|
-
cursor = conn.cursor()
|
|
495
|
-
cursor.execute("SELECT * FROM table")
|
|
496
|
-
results = cursor.fetchall()
|
|
497
|
-
|
|
498
|
-
# Return connection to pool (important!)
|
|
499
|
-
pool.return_connection(conn)
|
|
500
|
-
|
|
501
|
-
# Clean up when done
|
|
502
|
-
pool.close_all()
|
|
503
|
-
```
|
|
504
|
-
|
|
505
|
-
##### Using Context Manager (Recommended)
|
|
506
|
-
|
|
507
|
-
The context manager pattern ensures connections are automatically returned to the pool:
|
|
508
|
-
|
|
509
|
-
```python
|
|
510
|
-
from e6data_python_connector import ConnectionPool
|
|
511
|
-
|
|
512
|
-
pool = ConnectionPool(
|
|
513
|
-
min_size=2,
|
|
514
|
-
max_size=10,
|
|
515
|
-
host=host,
|
|
516
|
-
port=port,
|
|
517
|
-
username=username,
|
|
518
|
-
password=password,
|
|
519
|
-
database=database
|
|
520
|
-
)
|
|
521
|
-
|
|
522
|
-
# Connection automatically returned to pool after use
|
|
523
|
-
with pool.get_connection_context() as conn:
|
|
524
|
-
cursor = conn.cursor()
|
|
525
|
-
cursor.execute("SELECT * FROM table")
|
|
526
|
-
results = cursor.fetchall()
|
|
527
|
-
print(results)
|
|
528
|
-
```
|
|
529
|
-
|
|
530
|
-
##### Concurrent Query Execution
|
|
531
|
-
|
|
532
|
-
Connection pooling is especially beneficial for concurrent query execution:
|
|
533
|
-
|
|
534
|
-
```python
|
|
535
|
-
import concurrent.futures
|
|
536
|
-
from e6data_python_connector import ConnectionPool
|
|
537
|
-
|
|
538
|
-
def execute_query(pool, query_id, query):
|
|
539
|
-
"""Execute a query using a pooled connection."""
|
|
540
|
-
# Each thread will reuse its assigned connection
|
|
541
|
-
conn = pool.get_connection()
|
|
542
|
-
try:
|
|
543
|
-
cursor = conn.cursor()
|
|
544
|
-
cursor.execute(query)
|
|
545
|
-
results = cursor.fetchall()
|
|
546
|
-
return f"Query {query_id}: {len(results)} rows"
|
|
547
|
-
finally:
|
|
548
|
-
pool.return_connection(conn)
|
|
549
|
-
|
|
550
|
-
# Create pool
|
|
551
|
-
pool = ConnectionPool(
|
|
552
|
-
min_size=3,
|
|
553
|
-
max_size=10,
|
|
554
|
-
host=host,
|
|
555
|
-
port=port,
|
|
556
|
-
username=username,
|
|
557
|
-
password=password,
|
|
558
|
-
database=database
|
|
559
|
-
)
|
|
560
|
-
|
|
561
|
-
# Execute multiple queries concurrently
|
|
562
|
-
queries = [
|
|
563
|
-
"SELECT COUNT(*) FROM table1",
|
|
564
|
-
"SELECT AVG(value) FROM table2",
|
|
565
|
-
"SELECT MAX(date) FROM table3"
|
|
566
|
-
]
|
|
567
|
-
|
|
568
|
-
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
|
|
569
|
-
futures = [
|
|
570
|
-
executor.submit(execute_query, pool, i, query)
|
|
571
|
-
for i, query in enumerate(queries)
|
|
572
|
-
]
|
|
573
|
-
|
|
574
|
-
for future in concurrent.futures.as_completed(futures):
|
|
575
|
-
print(future.result())
|
|
576
|
-
|
|
577
|
-
# Clean up
|
|
578
|
-
pool.close_all()
|
|
579
|
-
```
|
|
580
|
-
|
|
581
|
-
##### Connection Pool Configuration
|
|
582
|
-
|
|
583
|
-
| Parameter | Type | Default | Description |
|
|
584
|
-
|-----------|------|---------|-------------|
|
|
585
|
-
| `min_size` | int | 2 | Minimum number of connections to maintain |
|
|
586
|
-
| `max_size` | int | 10 | Maximum number of connections in pool |
|
|
587
|
-
| `max_overflow` | int | 5 | Additional temporary connections allowed |
|
|
588
|
-
| `timeout` | float | 30.0 | Timeout for getting connection (seconds) |
|
|
589
|
-
| `recycle` | int | 3600 | Maximum connection age before recycling (seconds) |
|
|
590
|
-
| `debug` | bool | False | Enable debug logging for pool operations |
|
|
591
|
-
| `pre_ping` | bool | True | Check connection health before returning from pool |
|
|
592
|
-
|
|
593
|
-
##### Monitoring Pool Statistics
|
|
594
|
-
|
|
595
|
-
```python
|
|
596
|
-
# Get pool statistics
|
|
597
|
-
stats = pool.get_statistics()
|
|
598
|
-
print(f"Active connections: {stats['active_connections']}")
|
|
599
|
-
print(f"Idle connections: {stats['idle_connections']}")
|
|
600
|
-
print(f"Total requests: {stats['total_requests']}")
|
|
601
|
-
print(f"Failed connections: {stats['failed_connections']}")
|
|
602
|
-
```
|
|
603
|
-
|
|
604
|
-
##### When to Use Connection Pooling
|
|
605
|
-
|
|
606
|
-
Connection pooling is recommended when:
|
|
607
|
-
- Executing multiple queries concurrently
|
|
608
|
-
- Building web applications or APIs
|
|
609
|
-
- Running batch processing jobs
|
|
610
|
-
- Reducing connection overhead
|
|
611
|
-
- Improving application performance
|
|
612
|
-
|
|
613
|
-
##### Direct Connection Usage (Without Pool)
|
|
614
|
-
|
|
615
|
-
For simple, single-threaded applications, you can still use direct connections:
|
|
616
|
-
|
|
617
|
-
```python
|
|
618
|
-
from e6data_python_connector import Connection
|
|
619
|
-
|
|
620
|
-
conn = Connection(
|
|
621
|
-
host=host,
|
|
622
|
-
port=port,
|
|
623
|
-
username=username,
|
|
624
|
-
password=password,
|
|
625
|
-
database=database
|
|
626
|
-
)
|
|
627
|
-
|
|
628
|
-
cursor = conn.cursor()
|
|
629
|
-
cursor.execute("SELECT * FROM table")
|
|
630
|
-
results = cursor.fetchall()
|
|
631
|
-
conn.close()
|
|
632
|
-
```
|
|
633
|
-
|
|
634
|
-
#### Additional Connection Management Features
|
|
442
|
+
- Enable connection pooling for better resource utilization
|
|
635
443
|
- Automatic connection health monitoring
|
|
636
444
|
- Graceful connection recovery and retry logic
|
|
637
|
-
- Blue-green deployment support with automatic failover
|
|
638
445
|
|
|
639
446
|
See [TECH_DOC.md](TECH_DOC.md) for detailed technical documentation.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# e6data Python Connector
|
|
2
2
|
|
|
3
|
-

|
|
4
4
|
|
|
5
5
|
## Introduction
|
|
6
6
|
|
|
@@ -37,8 +37,6 @@ Use your e6data Email ID as the username and your access token as the password.
|
|
|
37
37
|
|
|
38
38
|
```python
|
|
39
39
|
from e6data_python_connector import Connection
|
|
40
|
-
# For connection pooling (recommended for concurrent operations)
|
|
41
|
-
from e6data_python_connector import ConnectionPool
|
|
42
40
|
|
|
43
41
|
username = '<username>' # Your e6data Email ID.
|
|
44
42
|
password = '<password>' # Access Token generated in the e6data console.
|
|
@@ -48,7 +46,6 @@ database = '<database>' # Database to perform the query on.
|
|
|
48
46
|
port = 80 # Port of the e6data engine.
|
|
49
47
|
catalog_name = '<catalog_name>'
|
|
50
48
|
|
|
51
|
-
# Single connection (for simple, single-threaded use)
|
|
52
49
|
conn = Connection(
|
|
53
50
|
host=host,
|
|
54
51
|
port=port,
|
|
@@ -56,17 +53,6 @@ conn = Connection(
|
|
|
56
53
|
database=database,
|
|
57
54
|
password=password
|
|
58
55
|
)
|
|
59
|
-
|
|
60
|
-
# Or use connection pool (for concurrent/multi-threaded use)
|
|
61
|
-
pool = ConnectionPool(
|
|
62
|
-
min_size=2,
|
|
63
|
-
max_size=10,
|
|
64
|
-
host=host,
|
|
65
|
-
port=port,
|
|
66
|
-
username=username,
|
|
67
|
-
database=database,
|
|
68
|
-
password=password
|
|
69
|
-
)
|
|
70
56
|
```
|
|
71
57
|
|
|
72
58
|
#### Connection Parameters
|
|
@@ -415,187 +401,8 @@ For detailed migration instructions, see the [Migration Guide](docs/zero-downtim
|
|
|
415
401
|
- Message size optimization for large queries
|
|
416
402
|
|
|
417
403
|
### Connection Management
|
|
418
|
-
|
|
419
|
-
#### Connection Pooling
|
|
420
|
-
|
|
421
|
-
The e6data Python connector now includes a built-in connection pool for efficient connection management and reuse across multiple threads. The `ConnectionPool` class provides:
|
|
422
|
-
|
|
423
|
-
- **Thread-safe connection reuse**: Each thread automatically reuses its assigned connection
|
|
424
|
-
- **Automatic lifecycle management**: Handles connection creation, health checks, and cleanup
|
|
425
|
-
- **Overflow connections**: Creates temporary connections when pool is exhausted
|
|
426
|
-
- **Connection health monitoring**: Automatic detection and replacement of broken connections
|
|
427
|
-
- **Statistics tracking**: Monitor pool usage and performance
|
|
428
|
-
|
|
429
|
-
##### Basic Connection Pool Usage
|
|
430
|
-
|
|
431
|
-
```python
|
|
432
|
-
from e6data_python_connector import ConnectionPool
|
|
433
|
-
|
|
434
|
-
# Create a connection pool
|
|
435
|
-
pool = ConnectionPool(
|
|
436
|
-
min_size=2, # Minimum connections to maintain
|
|
437
|
-
max_size=10, # Maximum connections in pool
|
|
438
|
-
max_overflow=5, # Additional temporary connections allowed
|
|
439
|
-
timeout=30.0, # Timeout for getting connection (seconds)
|
|
440
|
-
recycle=3600, # Maximum age before recycling (seconds)
|
|
441
|
-
debug=False, # Enable debug logging
|
|
442
|
-
pre_ping=True, # Check connection health before use
|
|
443
|
-
# Connection parameters
|
|
444
|
-
host=host,
|
|
445
|
-
port=port,
|
|
446
|
-
username=username,
|
|
447
|
-
password=password,
|
|
448
|
-
database=database,
|
|
449
|
-
catalog=catalog_name,
|
|
450
|
-
cluster_name=cluster_name,
|
|
451
|
-
secure=True
|
|
452
|
-
)
|
|
453
|
-
|
|
454
|
-
# Get connection and execute query
|
|
455
|
-
conn = pool.get_connection()
|
|
456
|
-
cursor = conn.cursor()
|
|
457
|
-
cursor.execute("SELECT * FROM table")
|
|
458
|
-
results = cursor.fetchall()
|
|
459
|
-
|
|
460
|
-
# Return connection to pool (important!)
|
|
461
|
-
pool.return_connection(conn)
|
|
462
|
-
|
|
463
|
-
# Clean up when done
|
|
464
|
-
pool.close_all()
|
|
465
|
-
```
|
|
466
|
-
|
|
467
|
-
##### Using Context Manager (Recommended)
|
|
468
|
-
|
|
469
|
-
The context manager pattern ensures connections are automatically returned to the pool:
|
|
470
|
-
|
|
471
|
-
```python
|
|
472
|
-
from e6data_python_connector import ConnectionPool
|
|
473
|
-
|
|
474
|
-
pool = ConnectionPool(
|
|
475
|
-
min_size=2,
|
|
476
|
-
max_size=10,
|
|
477
|
-
host=host,
|
|
478
|
-
port=port,
|
|
479
|
-
username=username,
|
|
480
|
-
password=password,
|
|
481
|
-
database=database
|
|
482
|
-
)
|
|
483
|
-
|
|
484
|
-
# Connection automatically returned to pool after use
|
|
485
|
-
with pool.get_connection_context() as conn:
|
|
486
|
-
cursor = conn.cursor()
|
|
487
|
-
cursor.execute("SELECT * FROM table")
|
|
488
|
-
results = cursor.fetchall()
|
|
489
|
-
print(results)
|
|
490
|
-
```
|
|
491
|
-
|
|
492
|
-
##### Concurrent Query Execution
|
|
493
|
-
|
|
494
|
-
Connection pooling is especially beneficial for concurrent query execution:
|
|
495
|
-
|
|
496
|
-
```python
|
|
497
|
-
import concurrent.futures
|
|
498
|
-
from e6data_python_connector import ConnectionPool
|
|
499
|
-
|
|
500
|
-
def execute_query(pool, query_id, query):
|
|
501
|
-
"""Execute a query using a pooled connection."""
|
|
502
|
-
# Each thread will reuse its assigned connection
|
|
503
|
-
conn = pool.get_connection()
|
|
504
|
-
try:
|
|
505
|
-
cursor = conn.cursor()
|
|
506
|
-
cursor.execute(query)
|
|
507
|
-
results = cursor.fetchall()
|
|
508
|
-
return f"Query {query_id}: {len(results)} rows"
|
|
509
|
-
finally:
|
|
510
|
-
pool.return_connection(conn)
|
|
511
|
-
|
|
512
|
-
# Create pool
|
|
513
|
-
pool = ConnectionPool(
|
|
514
|
-
min_size=3,
|
|
515
|
-
max_size=10,
|
|
516
|
-
host=host,
|
|
517
|
-
port=port,
|
|
518
|
-
username=username,
|
|
519
|
-
password=password,
|
|
520
|
-
database=database
|
|
521
|
-
)
|
|
522
|
-
|
|
523
|
-
# Execute multiple queries concurrently
|
|
524
|
-
queries = [
|
|
525
|
-
"SELECT COUNT(*) FROM table1",
|
|
526
|
-
"SELECT AVG(value) FROM table2",
|
|
527
|
-
"SELECT MAX(date) FROM table3"
|
|
528
|
-
]
|
|
529
|
-
|
|
530
|
-
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
|
|
531
|
-
futures = [
|
|
532
|
-
executor.submit(execute_query, pool, i, query)
|
|
533
|
-
for i, query in enumerate(queries)
|
|
534
|
-
]
|
|
535
|
-
|
|
536
|
-
for future in concurrent.futures.as_completed(futures):
|
|
537
|
-
print(future.result())
|
|
538
|
-
|
|
539
|
-
# Clean up
|
|
540
|
-
pool.close_all()
|
|
541
|
-
```
|
|
542
|
-
|
|
543
|
-
##### Connection Pool Configuration
|
|
544
|
-
|
|
545
|
-
| Parameter | Type | Default | Description |
|
|
546
|
-
|-----------|------|---------|-------------|
|
|
547
|
-
| `min_size` | int | 2 | Minimum number of connections to maintain |
|
|
548
|
-
| `max_size` | int | 10 | Maximum number of connections in pool |
|
|
549
|
-
| `max_overflow` | int | 5 | Additional temporary connections allowed |
|
|
550
|
-
| `timeout` | float | 30.0 | Timeout for getting connection (seconds) |
|
|
551
|
-
| `recycle` | int | 3600 | Maximum connection age before recycling (seconds) |
|
|
552
|
-
| `debug` | bool | False | Enable debug logging for pool operations |
|
|
553
|
-
| `pre_ping` | bool | True | Check connection health before returning from pool |
|
|
554
|
-
|
|
555
|
-
##### Monitoring Pool Statistics
|
|
556
|
-
|
|
557
|
-
```python
|
|
558
|
-
# Get pool statistics
|
|
559
|
-
stats = pool.get_statistics()
|
|
560
|
-
print(f"Active connections: {stats['active_connections']}")
|
|
561
|
-
print(f"Idle connections: {stats['idle_connections']}")
|
|
562
|
-
print(f"Total requests: {stats['total_requests']}")
|
|
563
|
-
print(f"Failed connections: {stats['failed_connections']}")
|
|
564
|
-
```
|
|
565
|
-
|
|
566
|
-
##### When to Use Connection Pooling
|
|
567
|
-
|
|
568
|
-
Connection pooling is recommended when:
|
|
569
|
-
- Executing multiple queries concurrently
|
|
570
|
-
- Building web applications or APIs
|
|
571
|
-
- Running batch processing jobs
|
|
572
|
-
- Reducing connection overhead
|
|
573
|
-
- Improving application performance
|
|
574
|
-
|
|
575
|
-
##### Direct Connection Usage (Without Pool)
|
|
576
|
-
|
|
577
|
-
For simple, single-threaded applications, you can still use direct connections:
|
|
578
|
-
|
|
579
|
-
```python
|
|
580
|
-
from e6data_python_connector import Connection
|
|
581
|
-
|
|
582
|
-
conn = Connection(
|
|
583
|
-
host=host,
|
|
584
|
-
port=port,
|
|
585
|
-
username=username,
|
|
586
|
-
password=password,
|
|
587
|
-
database=database
|
|
588
|
-
)
|
|
589
|
-
|
|
590
|
-
cursor = conn.cursor()
|
|
591
|
-
cursor.execute("SELECT * FROM table")
|
|
592
|
-
results = cursor.fetchall()
|
|
593
|
-
conn.close()
|
|
594
|
-
```
|
|
595
|
-
|
|
596
|
-
#### Additional Connection Management Features
|
|
404
|
+
- Enable connection pooling for better resource utilization
|
|
597
405
|
- Automatic connection health monitoring
|
|
598
406
|
- Graceful connection recovery and retry logic
|
|
599
|
-
- Blue-green deployment support with automatic failover
|
|
600
407
|
|
|
601
408
|
See [TECH_DOC.md](TECH_DOC.md) for detailed technical documentation.
|
|
@@ -370,7 +370,7 @@ class Connection(object):
|
|
|
370
370
|
self.__username = username
|
|
371
371
|
self.__password = password
|
|
372
372
|
self.database = database
|
|
373
|
-
self.cluster_name = cluster_name
|
|
373
|
+
self.cluster_name = cluster_name
|
|
374
374
|
self._session_id = None
|
|
375
375
|
self._host = host
|
|
376
376
|
self._port = port
|