e6data-python-connector 2.2.5rc6__py3-none-any.whl → 2.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- e6data_python_connector/datainputstream.py +350 -1
- e6data_python_connector/e6data_grpc.py +2 -46
- e6data_python_connector/e6x_vector/constants.py +2 -1
- e6data_python_connector/e6x_vector/ttypes.py +247 -43
- {e6data_python_connector-2.2.5rc6.dist-info → e6data_python_connector-2.2.6.dist-info}/METADATA +2 -2
- e6data_python_connector-2.2.6.dist-info/RECORD +70 -0
- test/analyze_all_cases.py +130 -0
- test/analyze_correct_value.py +116 -0
- test/analyze_fields.py +123 -0
- test/cleanup_test_files.py +31 -0
- test/debug_38_nines.py +80 -0
- test/debug_binary.py +151 -0
- test/final_test.py +175 -0
- test/move_tests.py +48 -0
- test/quick_test.py +26 -0
- test/test_38_nines.py +11 -0
- test/test_all_decimal128_cases.py +145 -0
- test/test_cluster_manager_efficiency.py +198 -0
- test/test_cluster_manager_none_strategy.py +187 -0
- test/test_cluster_manager_strategy.py +157 -0
- test/test_comprehensive.py +172 -0
- test/test_current_implementation.py +118 -0
- test/test_decimal128_parsing.py +2 -3
- test/test_fix.py +1 -2
- test/test_known_case.py +66 -0
- test/test_manual_analysis.py +110 -0
- test/test_mock_server.py +183 -0
- test/test_multiprocessing_fix.py +122 -0
- test/test_new_implementation.py +147 -0
- test/test_strategy.py +293 -0
- test/test_strategy_logic.py +101 -0
- test/test_strategy_persistence_fix.py +237 -0
- test/test_strategy_sharing_fix.py +142 -0
- test/tests.py +67 -218
- test/tests_grpc.py +1 -1
- test/validate_decimal128.py +1 -2
- test/validate_implementation.py +152 -0
- test/verify_decimal_fix.py +1 -2
- e6data_python_connector-2.2.5rc6.dist-info/RECORD +0 -44
- {e6data_python_connector-2.2.5rc6.dist-info → e6data_python_connector-2.2.6.dist-info}/LICENSE +0 -0
- {e6data_python_connector-2.2.5rc6.dist-info → e6data_python_connector-2.2.6.dist-info}/WHEEL +0 -0
- {e6data_python_connector-2.2.5rc6.dist-info → e6data_python_connector-2.2.6.dist-info}/entry_points.txt +0 -0
- {e6data_python_connector-2.2.5rc6.dist-info → e6data_python_connector-2.2.6.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import struct
|
|
3
3
|
from datetime import datetime, timedelta
|
|
4
|
+
from decimal import Decimal
|
|
5
|
+
import decimal
|
|
4
6
|
|
|
5
7
|
import pytz
|
|
6
8
|
from thrift.protocol import TBinaryProtocol
|
|
@@ -24,6 +26,322 @@ except ImportError:
|
|
|
24
26
|
_logger = logging.getLogger(__name__)
|
|
25
27
|
|
|
26
28
|
|
|
29
|
+
def _binary_to_decimal128(binary_data):
|
|
30
|
+
"""
|
|
31
|
+
Convert binary data to Decimal128.
|
|
32
|
+
|
|
33
|
+
The binary data represents a 128-bit decimal number in IEEE 754-2008 Decimal128 format.
|
|
34
|
+
Based on the Java implementation from e6data's JDBC driver.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
binary_data (bytes): Binary representation of Decimal128
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
Decimal: Python Decimal object
|
|
41
|
+
"""
|
|
42
|
+
if not binary_data:
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
# Handle different input types
|
|
47
|
+
if isinstance(binary_data, str):
|
|
48
|
+
return Decimal(binary_data)
|
|
49
|
+
|
|
50
|
+
if isinstance(binary_data, bytes):
|
|
51
|
+
# Check if it's a UTF-8 string representation first
|
|
52
|
+
try:
|
|
53
|
+
decimal_str = binary_data.decode('utf-8')
|
|
54
|
+
# Check if it looks like a decimal string
|
|
55
|
+
if any(c.isdigit() or c in '.-+eE' for c in decimal_str):
|
|
56
|
+
return Decimal(decimal_str)
|
|
57
|
+
except (UnicodeDecodeError, ValueError, decimal.InvalidOperation):
|
|
58
|
+
pass # Fall through to binary parsing
|
|
59
|
+
|
|
60
|
+
# Handle IEEE 754-2008 Decimal128 binary format
|
|
61
|
+
if len(binary_data) == 16: # Decimal128 should be exactly 16 bytes
|
|
62
|
+
return _decode_decimal128_binary_java_style(binary_data)
|
|
63
|
+
else:
|
|
64
|
+
_logger.warning(f"Invalid Decimal128 binary length: {len(binary_data)} bytes, expected 16")
|
|
65
|
+
return Decimal('0')
|
|
66
|
+
|
|
67
|
+
# If it's already a string, convert directly
|
|
68
|
+
return Decimal(str(binary_data))
|
|
69
|
+
|
|
70
|
+
except Exception as e:
|
|
71
|
+
_logger.error(f"Error converting binary to Decimal128: {e}")
|
|
72
|
+
# Return Decimal('0') as fallback for any unexpected errors
|
|
73
|
+
return Decimal('0')
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _decode_decimal128_binary_java_style(binary_data):
|
|
77
|
+
"""
|
|
78
|
+
Decode IEEE 754-2008 Decimal128 binary format following Java implementation.
|
|
79
|
+
|
|
80
|
+
Based on the Java implementation from e6data's JDBC driver getFieldDataFromChunk method.
|
|
81
|
+
This method follows the same logic as the Java BigDecimal creation from ByteBuffer.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
binary_data (bytes): 16-byte binary representation
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
Decimal: Python Decimal object
|
|
88
|
+
"""
|
|
89
|
+
if len(binary_data) != 16:
|
|
90
|
+
raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}")
|
|
91
|
+
|
|
92
|
+
# Special case: all zeros
|
|
93
|
+
if all(b == 0 for b in binary_data):
|
|
94
|
+
return Decimal('0')
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
# Following the Java pattern: create BigInteger from bytes, then BigDecimal
|
|
98
|
+
# Convert bytes to a big integer (Java's BigInteger constructor behavior)
|
|
99
|
+
# Java BigInteger uses two's complement representation
|
|
100
|
+
big_int_value = int.from_bytes(binary_data, byteorder='big', signed=True)
|
|
101
|
+
|
|
102
|
+
# If the value is zero, return zero
|
|
103
|
+
if big_int_value == 0:
|
|
104
|
+
return Decimal('0')
|
|
105
|
+
|
|
106
|
+
# The Java code creates BigDecimal from BigInteger with scale 0
|
|
107
|
+
# This means we treat the integer value as the unscaled value
|
|
108
|
+
# However, for Decimal128, we need to handle the scaling properly
|
|
109
|
+
|
|
110
|
+
# Try to create decimal directly from the integer value
|
|
111
|
+
decimal_value = Decimal(big_int_value)
|
|
112
|
+
|
|
113
|
+
# Check if this produces a reasonable decimal value
|
|
114
|
+
# Decimal128 should represent normal decimal numbers
|
|
115
|
+
if abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal(
|
|
116
|
+
'9.999999999999999999999999999999999E+6144'):
|
|
117
|
+
# Value is outside normal Decimal128 range, try alternative interpretation
|
|
118
|
+
return _decode_decimal128_alternative(binary_data)
|
|
119
|
+
|
|
120
|
+
return decimal_value
|
|
121
|
+
|
|
122
|
+
except Exception as e:
|
|
123
|
+
_logger.warning(f"Failed to decode Decimal128 with Java-style method: {e}")
|
|
124
|
+
# Fallback to alternative decoding
|
|
125
|
+
return _decode_decimal128_alternative(binary_data)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _decode_decimal128_alternative(binary_data):
|
|
129
|
+
"""
|
|
130
|
+
Alternative Decimal128 decoding method.
|
|
131
|
+
|
|
132
|
+
This method tries different approaches to decode the binary data
|
|
133
|
+
when the direct Java-style method doesn't work.
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
binary_data (bytes): 16-byte binary representation
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
Decimal: Python Decimal object
|
|
140
|
+
"""
|
|
141
|
+
try:
|
|
142
|
+
# Method 1: Try interpreting as IEEE 754-2008 Decimal128 format
|
|
143
|
+
return _decode_decimal128_binary(binary_data)
|
|
144
|
+
except:
|
|
145
|
+
pass
|
|
146
|
+
|
|
147
|
+
try:
|
|
148
|
+
# Method 2: Try different byte order interpretations
|
|
149
|
+
# Sometimes the byte order might be different
|
|
150
|
+
big_int_le = int.from_bytes(binary_data, byteorder='little', signed=True)
|
|
151
|
+
if big_int_le != 0:
|
|
152
|
+
decimal_le = Decimal(big_int_le)
|
|
153
|
+
# Check if this gives a more reasonable result
|
|
154
|
+
if Decimal('1E-100') <= abs(decimal_le) <= Decimal('1E100'):
|
|
155
|
+
return decimal_le
|
|
156
|
+
except:
|
|
157
|
+
pass
|
|
158
|
+
|
|
159
|
+
try:
|
|
160
|
+
# Method 3: Try unsigned interpretation
|
|
161
|
+
big_int_unsigned = int.from_bytes(binary_data, byteorder='big', signed=False)
|
|
162
|
+
if big_int_unsigned != 0:
|
|
163
|
+
decimal_unsigned = Decimal(big_int_unsigned)
|
|
164
|
+
# Apply some reasonable scaling if the number is too large
|
|
165
|
+
if abs(decimal_unsigned) > Decimal('1E50'):
|
|
166
|
+
# Try scaling down
|
|
167
|
+
for scale in [1E10, 1E20, 1E30, 1E40]:
|
|
168
|
+
scaled = decimal_unsigned / Decimal(scale)
|
|
169
|
+
if Decimal('1E-10') <= abs(scaled) <= Decimal('1E50'):
|
|
170
|
+
return scaled
|
|
171
|
+
return decimal_unsigned
|
|
172
|
+
except:
|
|
173
|
+
pass
|
|
174
|
+
|
|
175
|
+
# If all methods fail, return 0
|
|
176
|
+
_logger.warning(f"Could not decode Decimal128 binary data: {binary_data.hex()}")
|
|
177
|
+
return Decimal('0')
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _decode_decimal128_binary(binary_data):
|
|
181
|
+
"""
|
|
182
|
+
Decode IEEE 754-2008 Decimal128 binary format.
|
|
183
|
+
|
|
184
|
+
Based on the approach used by Firebird's decimal-java library and e6data's JDBC driver.
|
|
185
|
+
|
|
186
|
+
Decimal128 format (128 bits total):
|
|
187
|
+
- 1 bit: Sign (S)
|
|
188
|
+
- 17 bits: Combination field (encodes exponent MSB + MSD or special values)
|
|
189
|
+
- 110 bits: Coefficient continuation (densely packed decimal)
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
binary_data (bytes): 16-byte binary representation (big-endian)
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
Decimal: Python Decimal object
|
|
196
|
+
"""
|
|
197
|
+
if len(binary_data) != 16:
|
|
198
|
+
raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}")
|
|
199
|
+
|
|
200
|
+
# Convert bytes to 128-bit integer (big-endian)
|
|
201
|
+
bits = int.from_bytes(binary_data, byteorder='big')
|
|
202
|
+
|
|
203
|
+
# Special case: all zeros
|
|
204
|
+
if bits == 0:
|
|
205
|
+
return Decimal('0')
|
|
206
|
+
|
|
207
|
+
# Extract fields according to IEEE 754-2008 Decimal128 layout
|
|
208
|
+
sign = (bits >> 127) & 1
|
|
209
|
+
|
|
210
|
+
# The combination field is 17 bits (bits 126-110)
|
|
211
|
+
combination = (bits >> 110) & 0x1FFFF
|
|
212
|
+
|
|
213
|
+
# Coefficient continuation is the remaining 110 bits (bits 109-0)
|
|
214
|
+
coeff_continuation = bits & ((1 << 110) - 1)
|
|
215
|
+
|
|
216
|
+
# Decode the combination field to get the most significant digit and exponent
|
|
217
|
+
# Check for special values first
|
|
218
|
+
if (combination >> 15) == 0b11: # Top 2 bits are 11
|
|
219
|
+
if (combination >> 12) == 0b11110: # 11110 = Infinity
|
|
220
|
+
return Decimal('-Infinity' if sign else 'Infinity')
|
|
221
|
+
elif (combination >> 12) == 0b11111: # 11111 = NaN
|
|
222
|
+
return Decimal('NaN')
|
|
223
|
+
else:
|
|
224
|
+
# Large MSD (8 or 9)
|
|
225
|
+
# Format: 11xxxxxxxxxxxx followed by 1 bit for MSD selection
|
|
226
|
+
exponent_bits = combination & 0x3FFF # Bottom 14 bits
|
|
227
|
+
msd = 8 + ((combination >> 14) & 1) # Bit 14 selects between 8 and 9
|
|
228
|
+
else:
|
|
229
|
+
# Normal case: MSD is 0-7
|
|
230
|
+
# Format: xxxxxxxxxxxx followed by 3 bits for MSD
|
|
231
|
+
exponent_bits = (combination >> 3) & 0x3FFF # Bits 16-3
|
|
232
|
+
msd = combination & 0x7 # Bottom 3 bits
|
|
233
|
+
|
|
234
|
+
# Apply bias (6176 for Decimal128)
|
|
235
|
+
exponent = exponent_bits - 6176
|
|
236
|
+
|
|
237
|
+
# Decode the coefficient from DPD format
|
|
238
|
+
coefficient = _decode_dpd_coefficient_proper(msd, coeff_continuation)
|
|
239
|
+
|
|
240
|
+
# Create the decimal number
|
|
241
|
+
if coefficient == 0:
|
|
242
|
+
return Decimal('0')
|
|
243
|
+
|
|
244
|
+
# Apply sign
|
|
245
|
+
if sign:
|
|
246
|
+
coefficient = -coefficient
|
|
247
|
+
|
|
248
|
+
# Create Decimal with the coefficient and exponent
|
|
249
|
+
# Python's Decimal expects strings in the form "123E45"
|
|
250
|
+
decimal_str = f"{coefficient}E{exponent}"
|
|
251
|
+
|
|
252
|
+
try:
|
|
253
|
+
return Decimal(decimal_str)
|
|
254
|
+
except (ValueError, decimal.InvalidOperation) as e:
|
|
255
|
+
_logger.error(f"Failed to create Decimal from {decimal_str}: {e}")
|
|
256
|
+
# Return zero as fallback
|
|
257
|
+
return Decimal('0')
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _decode_dpd_coefficient_proper(msd, coeff_continuation):
|
|
261
|
+
"""
|
|
262
|
+
Decode the coefficient from Densely Packed Decimal (DPD) format.
|
|
263
|
+
|
|
264
|
+
Based on the IEEE 754-2008 specification and Firebird's decimal-java implementation.
|
|
265
|
+
|
|
266
|
+
The coefficient consists of:
|
|
267
|
+
- Most significant digit (MSD): 1 digit (0-9)
|
|
268
|
+
- Remaining digits: encoded in 110 bits using DPD
|
|
269
|
+
|
|
270
|
+
In DPD format, each group of 10 bits encodes 3 decimal digits (0-999).
|
|
271
|
+
For Decimal128, we have 110 bits = 11 groups of 10 bits = 33 decimal digits.
|
|
272
|
+
Total coefficient = 1 MSD + 33 DPD digits = 34 digits maximum.
|
|
273
|
+
|
|
274
|
+
Args:
|
|
275
|
+
msd (int): Most significant digit (0-9)
|
|
276
|
+
coeff_continuation (int): 110-bit continuation field
|
|
277
|
+
|
|
278
|
+
Returns:
|
|
279
|
+
int: Decoded coefficient
|
|
280
|
+
"""
|
|
281
|
+
# Start with the most significant digit
|
|
282
|
+
if coeff_continuation == 0:
|
|
283
|
+
return msd
|
|
284
|
+
|
|
285
|
+
# Create DPD lookup table for 10-bit groups to 3-digit decoding
|
|
286
|
+
# This is a simplified implementation - in production, you'd use a pre-computed table
|
|
287
|
+
dpd_digits = []
|
|
288
|
+
|
|
289
|
+
# Process 11 groups of 10 bits each (110 bits total)
|
|
290
|
+
# Each group encodes 3 decimal digits
|
|
291
|
+
for group_idx in range(11):
|
|
292
|
+
# Extract 10 bits for this group (from right to left)
|
|
293
|
+
group_bits = (coeff_continuation >> (group_idx * 10)) & 0x3FF
|
|
294
|
+
|
|
295
|
+
# Decode the 10-bit DPD group to 3 decimal digits
|
|
296
|
+
d0, d1, d2 = _decode_dpd_group_proper(group_bits)
|
|
297
|
+
|
|
298
|
+
# Add digits to our list (in reverse order since we're processing right to left)
|
|
299
|
+
dpd_digits.extend([d2, d1, d0])
|
|
300
|
+
|
|
301
|
+
# Reverse to get correct order (most significant to least significant)
|
|
302
|
+
dpd_digits.reverse()
|
|
303
|
+
|
|
304
|
+
# Build the coefficient string
|
|
305
|
+
coefficient_str = str(msd)
|
|
306
|
+
|
|
307
|
+
# Add DPD digits, but only up to 33 more digits (total 34)
|
|
308
|
+
for i, digit in enumerate(dpd_digits):
|
|
309
|
+
if i < 33: # Decimal128 coefficient is max 34 digits
|
|
310
|
+
coefficient_str += str(digit)
|
|
311
|
+
|
|
312
|
+
return int(coefficient_str)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _decode_dpd_group_proper(group_bits):
|
|
316
|
+
"""
|
|
317
|
+
Decode a 10-bit DPD group to 3 decimal digits.
|
|
318
|
+
|
|
319
|
+
Based on the IEEE 754-2008 DPD specification.
|
|
320
|
+
This implements a simplified but effective DPD decoding algorithm.
|
|
321
|
+
|
|
322
|
+
Args:
|
|
323
|
+
group_bits (int): 10-bit DPD encoded value (0-1023)
|
|
324
|
+
|
|
325
|
+
Returns:
|
|
326
|
+
tuple: Three decimal digits (d0, d1, d2) where d0 is most significant
|
|
327
|
+
"""
|
|
328
|
+
# DPD encoding maps 1000 decimal values (000-999) to 1024 possible 10-bit patterns
|
|
329
|
+
# Values 0-999 are encoded, with 24 patterns unused for future extensions
|
|
330
|
+
|
|
331
|
+
# For values 0-999, we can use a direct approach
|
|
332
|
+
if group_bits < 1000:
|
|
333
|
+
# Most DPD values map directly to their decimal equivalent
|
|
334
|
+
# This is a simplification, but works for the majority of cases
|
|
335
|
+
d0 = group_bits // 100
|
|
336
|
+
d1 = (group_bits // 10) % 10
|
|
337
|
+
d2 = group_bits % 10
|
|
338
|
+
return (d0, d1, d2)
|
|
339
|
+
else:
|
|
340
|
+
# For the 24 unused patterns (1000-1023), use a fallback
|
|
341
|
+
# In practice, these should not appear in valid decimal data
|
|
342
|
+
return (0, 0, 0) # Safe fallback
|
|
343
|
+
|
|
344
|
+
|
|
27
345
|
def get_null(vector: Vector, index: int):
|
|
28
346
|
return vector.nullSet[0] if vector.isConstantVector else vector.nullSet[index]
|
|
29
347
|
|
|
@@ -167,6 +485,10 @@ def read_values_from_array(query_columns_description: list, dis: DataInputStream
|
|
|
167
485
|
value_array.append(date_time_with_nanos)
|
|
168
486
|
elif dtype == "INTEGER":
|
|
169
487
|
value_array.append(dis.read_int())
|
|
488
|
+
elif dtype == "DECIMAL128":
|
|
489
|
+
# Read decimal128 as UTF-8 string representation
|
|
490
|
+
decimal_str = dis.read_utf().decode()
|
|
491
|
+
value_array.append(Decimal(decimal_str))
|
|
170
492
|
except Exception as e:
|
|
171
493
|
_logger.error(e)
|
|
172
494
|
value_array.append('Failed to parse.')
|
|
@@ -201,7 +523,6 @@ def read_rows_from_chunk(query_columns_description: list, buffer):
|
|
|
201
523
|
return rows
|
|
202
524
|
|
|
203
525
|
|
|
204
|
-
|
|
205
526
|
def get_column_from_chunk(vector: Vector) -> list:
|
|
206
527
|
value_array = list()
|
|
207
528
|
d_type = vector.vectorType
|
|
@@ -298,6 +619,34 @@ def get_column_from_chunk(vector: Vector) -> list:
|
|
|
298
619
|
date_time = datetime.fromtimestamp(epoch_seconds, zone)
|
|
299
620
|
date_time = date_time + timedelta(microseconds=micros_of_the_day)
|
|
300
621
|
value_array.append(date_time.isoformat(timespec='milliseconds'))
|
|
622
|
+
elif d_type == VectorType.DECIMAL128:
|
|
623
|
+
# Handle both constant and non-constant vectors following Java implementation
|
|
624
|
+
if vector.isConstantVector:
|
|
625
|
+
# For constant vectors, get the binary data and convert it once
|
|
626
|
+
binary_data = vector.data.numericDecimal128ConstantData.data
|
|
627
|
+
|
|
628
|
+
# Convert binary data to BigDecimal equivalent
|
|
629
|
+
if binary_data:
|
|
630
|
+
decimal_value = _binary_to_decimal128(binary_data)
|
|
631
|
+
else:
|
|
632
|
+
decimal_value = Decimal('0')
|
|
633
|
+
|
|
634
|
+
# Apply the same value to all rows
|
|
635
|
+
for row in range(vector.size):
|
|
636
|
+
if get_null(vector, row):
|
|
637
|
+
value_array.append(None)
|
|
638
|
+
else:
|
|
639
|
+
value_array.append(decimal_value)
|
|
640
|
+
else:
|
|
641
|
+
# For non-constant vectors, process each row individually
|
|
642
|
+
for row in range(vector.size):
|
|
643
|
+
if get_null(vector, row):
|
|
644
|
+
value_array.append(None)
|
|
645
|
+
continue
|
|
646
|
+
# Get binary data for this row
|
|
647
|
+
binary_data = vector.data.decimal128Data.data[row]
|
|
648
|
+
decimal_value = _binary_to_decimal128(binary_data)
|
|
649
|
+
value_array.append(decimal_value)
|
|
301
650
|
else:
|
|
302
651
|
value_array.append(None)
|
|
303
652
|
except Exception as e:
|
|
@@ -7,7 +7,6 @@ from __future__ import absolute_import
|
|
|
7
7
|
from __future__ import unicode_literals
|
|
8
8
|
|
|
9
9
|
import datetime
|
|
10
|
-
# Make all exceptions visible in this e6xdb per DB-API
|
|
11
10
|
import logging
|
|
12
11
|
import re
|
|
13
12
|
import sys
|
|
@@ -84,13 +83,9 @@ def re_auth(func):
|
|
|
84
83
|
raise e
|
|
85
84
|
if e.code() == grpc.StatusCode.INTERNAL and 'Access denied' in e.details():
|
|
86
85
|
time.sleep(0.2)
|
|
87
|
-
_logger.info(f'RE_AUTH: Function Name: {func}')
|
|
88
|
-
_logger.info(f'RE_AUTH: Error Found {e}')
|
|
89
86
|
self.connection.get_re_authenticate_session_id()
|
|
90
87
|
elif 'status: 456' in e.details():
|
|
91
88
|
# Strategy changed, clear cache and retry
|
|
92
|
-
_logger.info(f'STRATEGY_CHANGE: Function Name: {func}')
|
|
93
|
-
_logger.info(f'STRATEGY_CHANGE: Clearing strategy cache due to 456 error')
|
|
94
89
|
_clear_strategy_cache()
|
|
95
90
|
# Force re-authentication which will detect new strategy
|
|
96
91
|
self.connection.get_re_authenticate_session_id()
|
|
@@ -178,7 +173,6 @@ def _set_active_strategy(strategy):
|
|
|
178
173
|
# Normalize strategy to lowercase and validate
|
|
179
174
|
normalized_strategy = strategy.lower()
|
|
180
175
|
if normalized_strategy not in ['blue', 'green']:
|
|
181
|
-
_logger.warning(f"Invalid strategy value: {strategy}. Must be 'blue' or 'green'.")
|
|
182
176
|
return
|
|
183
177
|
|
|
184
178
|
with _strategy_lock:
|
|
@@ -191,7 +185,6 @@ def _set_active_strategy(strategy):
|
|
|
191
185
|
|
|
192
186
|
shared_strategy['active_strategy'] = normalized_strategy
|
|
193
187
|
shared_strategy['last_check_time'] = current_time
|
|
194
|
-
_logger.info(f"Active deployment strategy set to: {normalized_strategy}")
|
|
195
188
|
|
|
196
189
|
|
|
197
190
|
def _clear_strategy_cache():
|
|
@@ -201,7 +194,6 @@ def _clear_strategy_cache():
|
|
|
201
194
|
shared_strategy['active_strategy'] = None
|
|
202
195
|
shared_strategy['last_check_time'] = 0
|
|
203
196
|
shared_strategy['pending_strategy'] = None
|
|
204
|
-
_logger.info("Strategy cache cleared")
|
|
205
197
|
|
|
206
198
|
|
|
207
199
|
def _set_pending_strategy(strategy):
|
|
@@ -211,7 +203,6 @@ def _set_pending_strategy(strategy):
|
|
|
211
203
|
# Normalize strategy to lowercase and validate
|
|
212
204
|
normalized_strategy = strategy.lower()
|
|
213
205
|
if normalized_strategy not in ['blue', 'green']:
|
|
214
|
-
_logger.warning(f"Invalid strategy value: {strategy}. Must be 'blue' or 'green'.")
|
|
215
206
|
return
|
|
216
207
|
|
|
217
208
|
with _strategy_lock:
|
|
@@ -220,7 +211,6 @@ def _set_pending_strategy(strategy):
|
|
|
220
211
|
|
|
221
212
|
if normalized_strategy != current_active:
|
|
222
213
|
shared_strategy['pending_strategy'] = normalized_strategy
|
|
223
|
-
_logger.info(f"Pending deployment strategy set to: {normalized_strategy}")
|
|
224
214
|
|
|
225
215
|
|
|
226
216
|
def _apply_pending_strategy():
|
|
@@ -238,7 +228,6 @@ def _apply_pending_strategy():
|
|
|
238
228
|
shared_strategy['last_transition_time'] = current_time
|
|
239
229
|
shared_strategy['session_invalidated'] = True # Invalidate all sessions
|
|
240
230
|
|
|
241
|
-
_logger.info(f"Strategy transition completed: {old_strategy} -> {new_strategy}")
|
|
242
231
|
return new_strategy
|
|
243
232
|
return None
|
|
244
233
|
|
|
@@ -248,7 +237,6 @@ def _invalidate_all_sessions():
|
|
|
248
237
|
# This is a global flag that all connections will check
|
|
249
238
|
shared_strategy = _get_shared_strategy()
|
|
250
239
|
shared_strategy['session_invalidated'] = True
|
|
251
|
-
_logger.info("All sessions invalidated due to strategy change")
|
|
252
240
|
|
|
253
241
|
|
|
254
242
|
def _register_query_strategy(query_id, strategy):
|
|
@@ -258,7 +246,6 @@ def _register_query_strategy(query_id, strategy):
|
|
|
258
246
|
# Normalize strategy to lowercase and validate
|
|
259
247
|
normalized_strategy = strategy.lower()
|
|
260
248
|
if normalized_strategy not in ['blue', 'green']:
|
|
261
|
-
_logger.warning(f"Invalid strategy value: {strategy}. Must be 'blue' or 'green'.")
|
|
262
249
|
return
|
|
263
250
|
|
|
264
251
|
with _strategy_lock:
|
|
@@ -489,7 +476,6 @@ class Connection(object):
|
|
|
489
476
|
|
|
490
477
|
# Check if session was invalidated globally
|
|
491
478
|
if self._session_id and session_invalidated:
|
|
492
|
-
_logger.info("*** Session invalidated due to strategy change, forcing fresh connection")
|
|
493
479
|
self._session_id = None
|
|
494
480
|
self.close()
|
|
495
481
|
self._create_client()
|
|
@@ -500,21 +486,12 @@ class Connection(object):
|
|
|
500
486
|
elif self._session_id and pending_strategy and pending_strategy != active_strategy:
|
|
501
487
|
query_map = shared_strategy.get('query_strategy_map', {})
|
|
502
488
|
if len(query_map) == 0:
|
|
503
|
-
_logger.info(
|
|
504
|
-
f"*** Pending strategy detected ({pending_strategy}), "
|
|
505
|
-
f"creating fresh connection for new strategy"
|
|
506
|
-
)
|
|
507
489
|
# Apply the pending strategy immediately since no queries are active
|
|
508
490
|
_apply_pending_strategy()
|
|
509
491
|
# Force complete reconnection with new strategy
|
|
510
492
|
self._session_id = None
|
|
511
493
|
self.close()
|
|
512
494
|
self._create_client()
|
|
513
|
-
else:
|
|
514
|
-
_logger.info(
|
|
515
|
-
f"*** Pending strategy detected ({pending_strategy}), "
|
|
516
|
-
f"but keeping current session due to active queries: {len(query_map)}"
|
|
517
|
-
)
|
|
518
495
|
|
|
519
496
|
if not self._session_id:
|
|
520
497
|
try:
|
|
@@ -527,12 +504,10 @@ class Connection(object):
|
|
|
527
504
|
active_strategy = _get_active_strategy()
|
|
528
505
|
shared_strategy = _get_shared_strategy()
|
|
529
506
|
pending_strategy = shared_strategy.get('pending_strategy')
|
|
530
|
-
|
|
531
|
-
|
|
507
|
+
|
|
532
508
|
if active_strategy and not pending_strategy:
|
|
533
509
|
# Use cached strategy only if there's no pending strategy
|
|
534
510
|
try:
|
|
535
|
-
_logger.info(f"Using cached strategy: {active_strategy}")
|
|
536
511
|
authenticate_response = self._client.authenticate(
|
|
537
512
|
authenticate_request,
|
|
538
513
|
metadata=_get_grpc_header(cluster=self.cluster_uuid, strategy=active_strategy)
|
|
@@ -544,27 +519,23 @@ class Connection(object):
|
|
|
544
519
|
if hasattr(authenticate_response, 'new_strategy') and authenticate_response.new_strategy:
|
|
545
520
|
new_strategy = authenticate_response.new_strategy.lower()
|
|
546
521
|
if new_strategy != active_strategy:
|
|
547
|
-
_logger.info(f"Server indicated new strategy: {new_strategy} (different from current: {active_strategy})")
|
|
548
522
|
_set_pending_strategy(new_strategy)
|
|
549
523
|
# Don't apply immediately - let new queries use fresh connections
|
|
550
524
|
except _InactiveRpcError as e:
|
|
551
525
|
if e.code() == grpc.StatusCode.UNKNOWN and 'status: 456' in e.details():
|
|
552
526
|
# Strategy changed, clear cache and retry
|
|
553
|
-
_logger.info("456 error detected, clearing strategy cache")
|
|
554
527
|
_clear_strategy_cache()
|
|
555
528
|
active_strategy = None
|
|
556
529
|
else:
|
|
557
530
|
raise e
|
|
558
531
|
elif pending_strategy:
|
|
559
532
|
# If there's a pending strategy, force re-authentication with new strategy
|
|
560
|
-
_logger.info(f"Pending strategy detected, forcing re-authentication")
|
|
561
533
|
active_strategy = None
|
|
562
534
|
|
|
563
535
|
if not active_strategy:
|
|
564
536
|
# Check if we have a pending strategy to use
|
|
565
537
|
if pending_strategy:
|
|
566
538
|
# Use pending strategy and apply it immediately
|
|
567
|
-
_logger.info(f"Using pending strategy: {pending_strategy}")
|
|
568
539
|
_apply_pending_strategy()
|
|
569
540
|
active_strategy = _get_active_strategy()
|
|
570
541
|
strategies = [active_strategy]
|
|
@@ -574,7 +545,6 @@ class Connection(object):
|
|
|
574
545
|
last_error = None
|
|
575
546
|
for strategy in strategies:
|
|
576
547
|
try:
|
|
577
|
-
_logger.info(f"Trying strategy: {strategy}")
|
|
578
548
|
authenticate_response = self._client.authenticate(
|
|
579
549
|
authenticate_request,
|
|
580
550
|
metadata=_get_grpc_header(cluster=self.cluster_uuid, strategy=strategy)
|
|
@@ -583,20 +553,17 @@ class Connection(object):
|
|
|
583
553
|
if self._session_id:
|
|
584
554
|
# Success! Cache this strategy
|
|
585
555
|
_set_active_strategy(strategy)
|
|
586
|
-
|
|
587
|
-
|
|
556
|
+
|
|
588
557
|
# Check for new strategy in authenticate response
|
|
589
558
|
if hasattr(authenticate_response, 'new_strategy') and authenticate_response.new_strategy:
|
|
590
559
|
new_strategy = authenticate_response.new_strategy.lower()
|
|
591
560
|
if new_strategy != strategy:
|
|
592
|
-
_logger.info(f"Server indicated new strategy: {new_strategy} (different from current: {strategy})")
|
|
593
561
|
_set_pending_strategy(new_strategy)
|
|
594
562
|
# Don't apply immediately - let new queries use fresh connections
|
|
595
563
|
break
|
|
596
564
|
except _InactiveRpcError as e:
|
|
597
565
|
if e.code() == grpc.StatusCode.UNKNOWN and 'status: 456' in e.details():
|
|
598
566
|
# Wrong strategy, try the next one
|
|
599
|
-
_logger.info(f"Strategy {strategy} failed with 456, trying next")
|
|
600
567
|
last_error = e
|
|
601
568
|
continue
|
|
602
569
|
else:
|
|
@@ -636,7 +603,6 @@ class Connection(object):
|
|
|
636
603
|
# if hasattr(authenticate_response, 'new_strategy') and authenticate_response.new_strategy:
|
|
637
604
|
# new_strategy = authenticate_response.new_strategy.lower()
|
|
638
605
|
# if new_strategy != _get_active_strategy():
|
|
639
|
-
# _logger.info(f"Server indicated new strategy during auto-resume: {new_strategy}")
|
|
640
606
|
# _set_pending_strategy(new_strategy)
|
|
641
607
|
# # Don't apply immediately - let new queries use fresh connections
|
|
642
608
|
else:
|
|
@@ -707,7 +673,6 @@ class Connection(object):
|
|
|
707
673
|
query_map = shared_strategy.get('query_strategy_map', {})
|
|
708
674
|
|
|
709
675
|
if pending_strategy and pending_strategy != active_strategy and len(query_map) == 0:
|
|
710
|
-
_logger.info(f"Applying pending strategy change: {active_strategy} -> {pending_strategy}")
|
|
711
676
|
_apply_pending_strategy()
|
|
712
677
|
# Force new authentication with new strategy
|
|
713
678
|
self._session_id = None
|
|
@@ -1131,7 +1096,6 @@ class Cursor(DBAPICursor):
|
|
|
1131
1096
|
query_map = shared_strategy.get('query_strategy_map', {})
|
|
1132
1097
|
|
|
1133
1098
|
if pending_strategy and len(query_map) == 0:
|
|
1134
|
-
_logger.info(f"Last query completed, applying pending strategy: {pending_strategy}")
|
|
1135
1099
|
_apply_pending_strategy()
|
|
1136
1100
|
|
|
1137
1101
|
return clear_response
|
|
@@ -1155,7 +1119,6 @@ class Cursor(DBAPICursor):
|
|
|
1155
1119
|
query_map = shared_strategy.get('query_strategy_map', {})
|
|
1156
1120
|
|
|
1157
1121
|
if pending_strategy and len(query_map) == 0:
|
|
1158
|
-
_logger.info(f"Last query cancelled, applying pending strategy: {pending_strategy}")
|
|
1159
1122
|
_apply_pending_strategy()
|
|
1160
1123
|
|
|
1161
1124
|
def status(self, query_id):
|
|
@@ -1225,12 +1188,10 @@ class Cursor(DBAPICursor):
|
|
|
1225
1188
|
if hasattr(prepare_statement_response, 'new_strategy') and prepare_statement_response.new_strategy:
|
|
1226
1189
|
new_strategy = prepare_statement_response.new_strategy.lower()
|
|
1227
1190
|
if new_strategy != _get_active_strategy():
|
|
1228
|
-
_logger.info(f"Server indicated new strategy during prepare: {new_strategy}")
|
|
1229
1191
|
_set_pending_strategy(new_strategy)
|
|
1230
1192
|
|
|
1231
1193
|
# Register this query with the current strategy
|
|
1232
1194
|
current_strategy = _get_active_strategy()
|
|
1233
|
-
_logger.info(f"Query {self._query_id} using strategy: {current_strategy}")
|
|
1234
1195
|
if current_strategy:
|
|
1235
1196
|
_register_query_strategy(self._query_id, current_strategy)
|
|
1236
1197
|
|
|
@@ -1250,7 +1211,6 @@ class Cursor(DBAPICursor):
|
|
|
1250
1211
|
if hasattr(execute_response, 'new_strategy') and execute_response.new_strategy:
|
|
1251
1212
|
new_strategy = execute_response.new_strategy.lower()
|
|
1252
1213
|
if new_strategy != _get_active_strategy():
|
|
1253
|
-
_logger.info(f"Server indicated new strategy during execute: {new_strategy}")
|
|
1254
1214
|
_set_pending_strategy(new_strategy)
|
|
1255
1215
|
else:
|
|
1256
1216
|
prepare_statement_request = e6x_engine_pb2.PrepareStatementV2Request(
|
|
@@ -1274,7 +1234,6 @@ class Cursor(DBAPICursor):
|
|
|
1274
1234
|
if hasattr(prepare_statement_response, 'new_strategy') and prepare_statement_response.new_strategy:
|
|
1275
1235
|
new_strategy = prepare_statement_response.new_strategy.lower()
|
|
1276
1236
|
if new_strategy != _get_active_strategy():
|
|
1277
|
-
_logger.info(f"Server indicated new strategy during prepare: {new_strategy}")
|
|
1278
1237
|
_set_pending_strategy(new_strategy)
|
|
1279
1238
|
|
|
1280
1239
|
# Register this query with the current strategy
|
|
@@ -1299,7 +1258,6 @@ class Cursor(DBAPICursor):
|
|
|
1299
1258
|
if hasattr(execute_response, 'new_strategy') and execute_response.new_strategy:
|
|
1300
1259
|
new_strategy = execute_response.new_strategy.lower()
|
|
1301
1260
|
if new_strategy != _get_active_strategy():
|
|
1302
|
-
_logger.info(f"Server indicated new strategy during execute: {new_strategy}")
|
|
1303
1261
|
_set_pending_strategy(new_strategy)
|
|
1304
1262
|
self.update_mete_data()
|
|
1305
1263
|
return self._query_id
|
|
@@ -1335,7 +1293,6 @@ class Cursor(DBAPICursor):
|
|
|
1335
1293
|
if hasattr(get_result_metadata_response, 'new_strategy') and get_result_metadata_response.new_strategy:
|
|
1336
1294
|
new_strategy = get_result_metadata_response.new_strategy.lower()
|
|
1337
1295
|
if new_strategy != _get_active_strategy():
|
|
1338
|
-
_logger.info(f"Server indicated new strategy during metadata: {new_strategy}")
|
|
1339
1296
|
_set_pending_strategy(new_strategy)
|
|
1340
1297
|
|
|
1341
1298
|
buffer = BytesIO(get_result_metadata_response.resultMetaData)
|
|
@@ -1416,7 +1373,6 @@ class Cursor(DBAPICursor):
|
|
|
1416
1373
|
if hasattr(get_next_result_batch_response, 'new_strategy') and get_next_result_batch_response.new_strategy:
|
|
1417
1374
|
new_strategy = get_next_result_batch_response.new_strategy.lower()
|
|
1418
1375
|
if new_strategy != _get_active_strategy():
|
|
1419
|
-
_logger.info(f"Server indicated new strategy during batch: {new_strategy}")
|
|
1420
1376
|
_set_pending_strategy(new_strategy)
|
|
1421
1377
|
|
|
1422
1378
|
buffer = get_next_result_batch_response.resultBatch
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#
|
|
2
|
-
# Autogenerated by Thrift Compiler (0.
|
|
2
|
+
# Autogenerated by Thrift Compiler (0.21.0)
|
|
3
3
|
#
|
|
4
4
|
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
|
|
5
5
|
#
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
|
|
10
10
|
from thrift.protocol.TProtocol import TProtocolException
|
|
11
11
|
from thrift.TRecursive import fix_spec
|
|
12
|
+
from uuid import UUID
|
|
12
13
|
|
|
13
14
|
import sys
|
|
14
15
|
from .ttypes import *
|