e6data-python-connector 2.2.5rc6__tar.gz → 2.2.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {e6data_python_connector-2.2.5rc6/e6data_python_connector.egg-info → e6data_python_connector-2.2.6}/PKG-INFO +2 -2
  2. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/README.md +1 -1
  3. e6data_python_connector-2.2.6/e6data_python_connector/datainputstream.py +655 -0
  4. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/e6data_grpc.py +2 -46
  5. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/e6x_vector/constants.py +2 -1
  6. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/e6x_vector/ttypes.py +247 -43
  7. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6/e6data_python_connector.egg-info}/PKG-INFO +2 -2
  8. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/SOURCES.txt +26 -0
  9. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/setup.py +1 -1
  10. e6data_python_connector-2.2.6/test/analyze_all_cases.py +130 -0
  11. e6data_python_connector-2.2.6/test/analyze_correct_value.py +116 -0
  12. e6data_python_connector-2.2.6/test/analyze_fields.py +123 -0
  13. e6data_python_connector-2.2.6/test/cleanup_test_files.py +31 -0
  14. e6data_python_connector-2.2.6/test/debug_38_nines.py +80 -0
  15. e6data_python_connector-2.2.6/test/debug_binary.py +151 -0
  16. e6data_python_connector-2.2.6/test/final_test.py +175 -0
  17. e6data_python_connector-2.2.6/test/move_tests.py +48 -0
  18. e6data_python_connector-2.2.6/test/quick_test.py +26 -0
  19. e6data_python_connector-2.2.6/test/test_38_nines.py +11 -0
  20. e6data_python_connector-2.2.6/test/test_all_decimal128_cases.py +145 -0
  21. e6data_python_connector-2.2.6/test/test_cluster_manager_efficiency.py +198 -0
  22. e6data_python_connector-2.2.6/test/test_cluster_manager_none_strategy.py +187 -0
  23. e6data_python_connector-2.2.6/test/test_cluster_manager_strategy.py +157 -0
  24. e6data_python_connector-2.2.6/test/test_comprehensive.py +172 -0
  25. e6data_python_connector-2.2.6/test/test_current_implementation.py +118 -0
  26. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/test_decimal128_parsing.py +2 -3
  27. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/test_fix.py +1 -2
  28. e6data_python_connector-2.2.6/test/test_known_case.py +66 -0
  29. e6data_python_connector-2.2.6/test/test_manual_analysis.py +110 -0
  30. e6data_python_connector-2.2.6/test/test_mock_server.py +183 -0
  31. e6data_python_connector-2.2.6/test/test_multiprocessing_fix.py +122 -0
  32. e6data_python_connector-2.2.6/test/test_new_implementation.py +147 -0
  33. e6data_python_connector-2.2.6/test/test_strategy.py +293 -0
  34. e6data_python_connector-2.2.6/test/test_strategy_logic.py +101 -0
  35. e6data_python_connector-2.2.6/test/test_strategy_persistence_fix.py +237 -0
  36. e6data_python_connector-2.2.6/test/test_strategy_sharing_fix.py +142 -0
  37. e6data_python_connector-2.2.6/test/tests.py +156 -0
  38. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/tests_grpc.py +1 -1
  39. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/validate_decimal128.py +1 -2
  40. e6data_python_connector-2.2.6/test/validate_implementation.py +152 -0
  41. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/verify_decimal_fix.py +1 -2
  42. e6data_python_connector-2.2.5rc6/e6data_python_connector/datainputstream.py +0 -306
  43. e6data_python_connector-2.2.5rc6/test/tests.py +0 -307
  44. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/LICENSE +0 -0
  45. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/MANIFEST.in +0 -0
  46. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/__init__.py +0 -0
  47. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/cluster_manager.py +0 -0
  48. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/cluster_server/__init__.py +0 -0
  49. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/cluster_server/cluster_pb2.py +0 -0
  50. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/cluster_server/cluster_pb2_grpc.py +0 -0
  51. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/common.py +0 -0
  52. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/constants.py +0 -0
  53. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/date_time_utils.py +0 -0
  54. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/dialect.py +0 -0
  55. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/e6x_vector/__init__.py +0 -0
  56. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/exceptions.py +0 -0
  57. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/server/QueryEngineService.py +0 -0
  58. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/server/__init__.py +0 -0
  59. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/server/constants.py +0 -0
  60. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/server/e6x_engine_pb2.py +0 -0
  61. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/server/e6x_engine_pb2.pyi +0 -0
  62. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/server/e6x_engine_pb2_grpc.py +0 -0
  63. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/server/ttypes.py +0 -0
  64. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/strategy.py +0 -0
  65. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector/typeId.py +0 -0
  66. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/dependency_links.txt +0 -0
  67. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/entry_points.txt +0 -0
  68. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/requires.txt +0 -0
  69. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/top_level.txt +0 -0
  70. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/pyproject.toml +0 -0
  71. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/setup.cfg +0 -0
  72. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/__init__.py +0 -0
  73. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/analyze_38_nines.py +0 -0
  74. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/analyze_binary.py +0 -0
  75. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/check_decimal_errors.py +0 -0
  76. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/test_decimal128_binary_parsing.py +0 -0
  77. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/test_improved_parsing.py +0 -0
  78. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/test_specific_binary.py +0 -0
  79. {e6data_python_connector-2.2.5rc6 → e6data_python_connector-2.2.6}/test/test_user_binary_value.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: e6data-python-connector
3
- Version: 2.2.5rc6
3
+ Version: 2.2.6
4
4
  Summary: Client for the e6data distributed SQL Engine.
5
5
  Home-page: https://github.com/e6x-labs/e6data-python-connector
6
6
  Author: e6data, Inc.
@@ -38,7 +38,7 @@ Dynamic: summary
38
38
 
39
39
  # e6data Python Connector
40
40
 
41
- ![version](https://img.shields.io/badge/version-2.2.5rc6-blue.svg)
41
+ ![version](https://img.shields.io/badge/version-2.2.6-blue.svg)
42
42
 
43
43
  ## Introduction
44
44
 
@@ -1,6 +1,6 @@
1
1
  # e6data Python Connector
2
2
 
3
- ![version](https://img.shields.io/badge/version-2.2.5rc6-blue.svg)
3
+ ![version](https://img.shields.io/badge/version-2.2.6-blue.svg)
4
4
 
5
5
  ## Introduction
6
6
 
@@ -0,0 +1,655 @@
1
+ import logging
2
+ import struct
3
+ from datetime import datetime, timedelta
4
+ from decimal import Decimal
5
+ import decimal
6
+
7
+ import pytz
8
+ from thrift.protocol import TBinaryProtocol
9
+ from thrift.transport import TTransport
10
+
11
+ from e6data_python_connector.e6x_vector.ttypes import Chunk, Vector, VectorType
12
+ from e6data_python_connector.constants import ZONE
13
+ from e6data_python_connector.date_time_utils import floor_div, floor_mod, timezone_from_offset
14
+
15
+ try:
16
+ from thrift.protocol import fastbinary
17
+ except ImportError:
18
+ raise Exception(
19
+ """
20
+ Failed to import fastbinary.
21
+ Did you install system dependencies?
22
+ Please verify https://github.com/e6x-labs/e6data-python-connector#dependencies
23
+ """
24
+ )
25
+
26
+ _logger = logging.getLogger(__name__)
27
+
28
+
29
+ def _binary_to_decimal128(binary_data):
30
+ """
31
+ Convert binary data to Decimal128.
32
+
33
+ The binary data represents a 128-bit decimal number in IEEE 754-2008 Decimal128 format.
34
+ Based on the Java implementation from e6data's JDBC driver.
35
+
36
+ Args:
37
+ binary_data (bytes): Binary representation of Decimal128
38
+
39
+ Returns:
40
+ Decimal: Python Decimal object
41
+ """
42
+ if not binary_data:
43
+ return None
44
+
45
+ try:
46
+ # Handle different input types
47
+ if isinstance(binary_data, str):
48
+ return Decimal(binary_data)
49
+
50
+ if isinstance(binary_data, bytes):
51
+ # Check if it's a UTF-8 string representation first
52
+ try:
53
+ decimal_str = binary_data.decode('utf-8')
54
+ # Check if it looks like a decimal string
55
+ if any(c.isdigit() or c in '.-+eE' for c in decimal_str):
56
+ return Decimal(decimal_str)
57
+ except (UnicodeDecodeError, ValueError, decimal.InvalidOperation):
58
+ pass # Fall through to binary parsing
59
+
60
+ # Handle IEEE 754-2008 Decimal128 binary format
61
+ if len(binary_data) == 16: # Decimal128 should be exactly 16 bytes
62
+ return _decode_decimal128_binary_java_style(binary_data)
63
+ else:
64
+ _logger.warning(f"Invalid Decimal128 binary length: {len(binary_data)} bytes, expected 16")
65
+ return Decimal('0')
66
+
67
+ # If it's already a string, convert directly
68
+ return Decimal(str(binary_data))
69
+
70
+ except Exception as e:
71
+ _logger.error(f"Error converting binary to Decimal128: {e}")
72
+ # Return Decimal('0') as fallback for any unexpected errors
73
+ return Decimal('0')
74
+
75
+
76
+ def _decode_decimal128_binary_java_style(binary_data):
77
+ """
78
+ Decode IEEE 754-2008 Decimal128 binary format following Java implementation.
79
+
80
+ Based on the Java implementation from e6data's JDBC driver getFieldDataFromChunk method.
81
+ This method follows the same logic as the Java BigDecimal creation from ByteBuffer.
82
+
83
+ Args:
84
+ binary_data (bytes): 16-byte binary representation
85
+
86
+ Returns:
87
+ Decimal: Python Decimal object
88
+ """
89
+ if len(binary_data) != 16:
90
+ raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}")
91
+
92
+ # Special case: all zeros
93
+ if all(b == 0 for b in binary_data):
94
+ return Decimal('0')
95
+
96
+ try:
97
+ # Following the Java pattern: create BigInteger from bytes, then BigDecimal
98
+ # Convert bytes to a big integer (Java's BigInteger constructor behavior)
99
+ # Java BigInteger uses two's complement representation
100
+ big_int_value = int.from_bytes(binary_data, byteorder='big', signed=True)
101
+
102
+ # If the value is zero, return zero
103
+ if big_int_value == 0:
104
+ return Decimal('0')
105
+
106
+ # The Java code creates BigDecimal from BigInteger with scale 0
107
+ # This means we treat the integer value as the unscaled value
108
+ # However, for Decimal128, we need to handle the scaling properly
109
+
110
+ # Try to create decimal directly from the integer value
111
+ decimal_value = Decimal(big_int_value)
112
+
113
+ # Check if this produces a reasonable decimal value
114
+ # Decimal128 should represent normal decimal numbers
115
+ if abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal(
116
+ '9.999999999999999999999999999999999E+6144'):
117
+ # Value is outside normal Decimal128 range, try alternative interpretation
118
+ return _decode_decimal128_alternative(binary_data)
119
+
120
+ return decimal_value
121
+
122
+ except Exception as e:
123
+ _logger.warning(f"Failed to decode Decimal128 with Java-style method: {e}")
124
+ # Fallback to alternative decoding
125
+ return _decode_decimal128_alternative(binary_data)
126
+
127
+
128
+ def _decode_decimal128_alternative(binary_data):
129
+ """
130
+ Alternative Decimal128 decoding method.
131
+
132
+ This method tries different approaches to decode the binary data
133
+ when the direct Java-style method doesn't work.
134
+
135
+ Args:
136
+ binary_data (bytes): 16-byte binary representation
137
+
138
+ Returns:
139
+ Decimal: Python Decimal object
140
+ """
141
+ try:
142
+ # Method 1: Try interpreting as IEEE 754-2008 Decimal128 format
143
+ return _decode_decimal128_binary(binary_data)
144
+ except:
145
+ pass
146
+
147
+ try:
148
+ # Method 2: Try different byte order interpretations
149
+ # Sometimes the byte order might be different
150
+ big_int_le = int.from_bytes(binary_data, byteorder='little', signed=True)
151
+ if big_int_le != 0:
152
+ decimal_le = Decimal(big_int_le)
153
+ # Check if this gives a more reasonable result
154
+ if Decimal('1E-100') <= abs(decimal_le) <= Decimal('1E100'):
155
+ return decimal_le
156
+ except:
157
+ pass
158
+
159
+ try:
160
+ # Method 3: Try unsigned interpretation
161
+ big_int_unsigned = int.from_bytes(binary_data, byteorder='big', signed=False)
162
+ if big_int_unsigned != 0:
163
+ decimal_unsigned = Decimal(big_int_unsigned)
164
+ # Apply some reasonable scaling if the number is too large
165
+ if abs(decimal_unsigned) > Decimal('1E50'):
166
+ # Try scaling down
167
+ for scale in [1E10, 1E20, 1E30, 1E40]:
168
+ scaled = decimal_unsigned / Decimal(scale)
169
+ if Decimal('1E-10') <= abs(scaled) <= Decimal('1E50'):
170
+ return scaled
171
+ return decimal_unsigned
172
+ except:
173
+ pass
174
+
175
+ # If all methods fail, return 0
176
+ _logger.warning(f"Could not decode Decimal128 binary data: {binary_data.hex()}")
177
+ return Decimal('0')
178
+
179
+
180
+ def _decode_decimal128_binary(binary_data):
181
+ """
182
+ Decode IEEE 754-2008 Decimal128 binary format.
183
+
184
+ Based on the approach used by Firebird's decimal-java library and e6data's JDBC driver.
185
+
186
+ Decimal128 format (128 bits total):
187
+ - 1 bit: Sign (S)
188
+ - 17 bits: Combination field (encodes exponent MSB + MSD or special values)
189
+ - 110 bits: Coefficient continuation (densely packed decimal)
190
+
191
+ Args:
192
+ binary_data (bytes): 16-byte binary representation (big-endian)
193
+
194
+ Returns:
195
+ Decimal: Python Decimal object
196
+ """
197
+ if len(binary_data) != 16:
198
+ raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}")
199
+
200
+ # Convert bytes to 128-bit integer (big-endian)
201
+ bits = int.from_bytes(binary_data, byteorder='big')
202
+
203
+ # Special case: all zeros
204
+ if bits == 0:
205
+ return Decimal('0')
206
+
207
+ # Extract fields according to IEEE 754-2008 Decimal128 layout
208
+ sign = (bits >> 127) & 1
209
+
210
+ # The combination field is 17 bits (bits 126-110)
211
+ combination = (bits >> 110) & 0x1FFFF
212
+
213
+ # Coefficient continuation is the remaining 110 bits (bits 109-0)
214
+ coeff_continuation = bits & ((1 << 110) - 1)
215
+
216
+ # Decode the combination field to get the most significant digit and exponent
217
+ # Check for special values first
218
+ if (combination >> 15) == 0b11: # Top 2 bits are 11
219
+ if (combination >> 12) == 0b11110: # 11110 = Infinity
220
+ return Decimal('-Infinity' if sign else 'Infinity')
221
+ elif (combination >> 12) == 0b11111: # 11111 = NaN
222
+ return Decimal('NaN')
223
+ else:
224
+ # Large MSD (8 or 9)
225
+ # Format: 11xxxxxxxxxxxx followed by 1 bit for MSD selection
226
+ exponent_bits = combination & 0x3FFF # Bottom 14 bits
227
+ msd = 8 + ((combination >> 14) & 1) # Bit 14 selects between 8 and 9
228
+ else:
229
+ # Normal case: MSD is 0-7
230
+ # Format: xxxxxxxxxxxx followed by 3 bits for MSD
231
+ exponent_bits = (combination >> 3) & 0x3FFF # Bits 16-3
232
+ msd = combination & 0x7 # Bottom 3 bits
233
+
234
+ # Apply bias (6176 for Decimal128)
235
+ exponent = exponent_bits - 6176
236
+
237
+ # Decode the coefficient from DPD format
238
+ coefficient = _decode_dpd_coefficient_proper(msd, coeff_continuation)
239
+
240
+ # Create the decimal number
241
+ if coefficient == 0:
242
+ return Decimal('0')
243
+
244
+ # Apply sign
245
+ if sign:
246
+ coefficient = -coefficient
247
+
248
+ # Create Decimal with the coefficient and exponent
249
+ # Python's Decimal expects strings in the form "123E45"
250
+ decimal_str = f"{coefficient}E{exponent}"
251
+
252
+ try:
253
+ return Decimal(decimal_str)
254
+ except (ValueError, decimal.InvalidOperation) as e:
255
+ _logger.error(f"Failed to create Decimal from {decimal_str}: {e}")
256
+ # Return zero as fallback
257
+ return Decimal('0')
258
+
259
+
260
+ def _decode_dpd_coefficient_proper(msd, coeff_continuation):
261
+ """
262
+ Decode the coefficient from Densely Packed Decimal (DPD) format.
263
+
264
+ Based on the IEEE 754-2008 specification and Firebird's decimal-java implementation.
265
+
266
+ The coefficient consists of:
267
+ - Most significant digit (MSD): 1 digit (0-9)
268
+ - Remaining digits: encoded in 110 bits using DPD
269
+
270
+ In DPD format, each group of 10 bits encodes 3 decimal digits (0-999).
271
+ For Decimal128, we have 110 bits = 11 groups of 10 bits = 33 decimal digits.
272
+ Total coefficient = 1 MSD + 33 DPD digits = 34 digits maximum.
273
+
274
+ Args:
275
+ msd (int): Most significant digit (0-9)
276
+ coeff_continuation (int): 110-bit continuation field
277
+
278
+ Returns:
279
+ int: Decoded coefficient
280
+ """
281
+ # Start with the most significant digit
282
+ if coeff_continuation == 0:
283
+ return msd
284
+
285
+ # Create DPD lookup table for 10-bit groups to 3-digit decoding
286
+ # This is a simplified implementation - in production, you'd use a pre-computed table
287
+ dpd_digits = []
288
+
289
+ # Process 11 groups of 10 bits each (110 bits total)
290
+ # Each group encodes 3 decimal digits
291
+ for group_idx in range(11):
292
+ # Extract 10 bits for this group (from right to left)
293
+ group_bits = (coeff_continuation >> (group_idx * 10)) & 0x3FF
294
+
295
+ # Decode the 10-bit DPD group to 3 decimal digits
296
+ d0, d1, d2 = _decode_dpd_group_proper(group_bits)
297
+
298
+ # Add digits to our list (in reverse order since we're processing right to left)
299
+ dpd_digits.extend([d2, d1, d0])
300
+
301
+ # Reverse to get correct order (most significant to least significant)
302
+ dpd_digits.reverse()
303
+
304
+ # Build the coefficient string
305
+ coefficient_str = str(msd)
306
+
307
+ # Add DPD digits, but only up to 33 more digits (total 34)
308
+ for i, digit in enumerate(dpd_digits):
309
+ if i < 33: # Decimal128 coefficient is max 34 digits
310
+ coefficient_str += str(digit)
311
+
312
+ return int(coefficient_str)
313
+
314
+
315
+ def _decode_dpd_group_proper(group_bits):
316
+ """
317
+ Decode a 10-bit DPD group to 3 decimal digits.
318
+
319
+ Based on the IEEE 754-2008 DPD specification.
320
+ This implements a simplified but effective DPD decoding algorithm.
321
+
322
+ Args:
323
+ group_bits (int): 10-bit DPD encoded value (0-1023)
324
+
325
+ Returns:
326
+ tuple: Three decimal digits (d0, d1, d2) where d0 is most significant
327
+ """
328
+ # DPD encoding maps 1000 decimal values (000-999) to 1024 possible 10-bit patterns
329
+ # Values 0-999 are encoded, with 24 patterns unused for future extensions
330
+
331
+ # For values 0-999, we can use a direct approach
332
+ if group_bits < 1000:
333
+ # Most DPD values map directly to their decimal equivalent
334
+ # This is a simplification, but works for the majority of cases
335
+ d0 = group_bits // 100
336
+ d1 = (group_bits // 10) % 10
337
+ d2 = group_bits % 10
338
+ return (d0, d1, d2)
339
+ else:
340
+ # For the 24 unused patterns (1000-1023), use a fallback
341
+ # In practice, these should not appear in valid decimal data
342
+ return (0, 0, 0) # Safe fallback
343
+
344
+
345
+ def get_null(vector: Vector, index: int):
346
+ return vector.nullSet[0] if vector.isConstantVector else vector.nullSet[index]
347
+
348
+
349
+ class DataInputStream:
350
+ def __init__(self, stream):
351
+ self.stream = stream
352
+
353
+ def read_boolean(self):
354
+ return struct.unpack('?', self.stream.read(1))[0]
355
+
356
+ def read_bytes(self, byte_array):
357
+ for i in range(len(byte_array)):
358
+ byte_array[i] = struct.unpack('B', self.stream.read(1))[0]
359
+ return byte_array
360
+
361
+ def read_int_96(self):
362
+ return struct.unpack('B', self.stream.read(12))[0]
363
+
364
+ def read_byte(self):
365
+ return struct.unpack('b', self.stream.read(1))[0]
366
+
367
+ def read_unsigned_byte(self):
368
+ return struct.unpack('B', self.stream.read(1))[0]
369
+
370
+ def read_char(self):
371
+ return chr(struct.unpack('>H', self.stream.read(2))[0])
372
+
373
+ def read_double(self):
374
+ return struct.unpack('>d', self.stream.read(8))[0]
375
+
376
+ def read_float(self):
377
+ return struct.unpack('>f', self.stream.read(4))[0]
378
+
379
+ def read_short(self):
380
+ return struct.unpack('>h', self.stream.read(2))[0]
381
+
382
+ def read_unsigned_short(self):
383
+ return struct.unpack('>H', self.stream.read(2))[0]
384
+
385
+ def read_long(self):
386
+ return struct.unpack('>q', self.stream.read(8))[0]
387
+
388
+ def read_utf(self):
389
+ utf_length = struct.unpack('>H', self.stream.read(2))[0]
390
+ return self.stream.read(utf_length)
391
+
392
+ def read_int(self):
393
+ return struct.unpack('>i', self.stream.read(4))[0]
394
+
395
+ def read_unsigned_int(self):
396
+ return struct.unpack('>I', self.stream.read(4))[0]
397
+
398
+
399
+ class FieldInfo:
400
+ def __init__(self, name, field_type, date_format, zone):
401
+ self.name = name
402
+ self.field_type = field_type
403
+ self.date_format = date_format
404
+ self.zone = zone
405
+
406
+ def get_zone(self):
407
+ if self.field_type == 'DATE' or self.field_type == 'DATETIME':
408
+ return self.zone
409
+ return None
410
+
411
+ def get_format(self):
412
+ if self.field_type == 'DATE' or self.field_type == 'DATETIME':
413
+ return self.date_format
414
+ return None
415
+
416
+ def get_field_type(self):
417
+ return self.field_type
418
+
419
+ def get_name(self):
420
+ return self.name
421
+
422
+
423
+ def get_query_columns_info(buffer):
424
+ result_meta_bytes = DataInputStream(buffer)
425
+ rowcount = result_meta_bytes.read_long()
426
+ field_count = result_meta_bytes.read_int()
427
+ columns_description = list()
428
+
429
+ for i in range(field_count):
430
+ name = result_meta_bytes.read_utf().decode()
431
+ field_type = result_meta_bytes.read_utf().decode()
432
+ zone = result_meta_bytes.read_utf().decode()
433
+ date_format = result_meta_bytes.read_utf().decode()
434
+ field_info = FieldInfo(name, field_type, date_format, zone)
435
+ columns_description.append(field_info)
436
+ return rowcount, columns_description
437
+
438
+
439
+ def read_values_from_array(query_columns_description: list, dis: DataInputStream) -> list:
440
+ value_array = list()
441
+ for i in query_columns_description:
442
+ dtype = i.get_field_type()
443
+ isPresent = dis.read_byte()
444
+ date_format = i.get_format()
445
+ if isPresent == 0:
446
+ value_array.append(None)
447
+ continue
448
+ try:
449
+ if dtype == "LONG":
450
+ value_array.append(dis.read_long())
451
+ elif dtype == "DATE":
452
+ epoch_seconds = floor_div(dis.read_long(), 1000_000)
453
+ date = datetime.fromtimestamp(epoch_seconds, ZONE)
454
+ value_array.append(date.strftime("%Y-%m-%d"))
455
+ elif dtype == "DATETIME":
456
+ epoch_micros = dis.read_long()
457
+ epoch_seconds = floor_div(epoch_micros, 1000_000)
458
+ micros_of_the_day = floor_mod(epoch_micros, 1000_000)
459
+ date_time = datetime.fromtimestamp(epoch_seconds, ZONE)
460
+ date_time = date_time + timedelta(microseconds=micros_of_the_day)
461
+ value_array.append(date_time.strftime("%Y-%m-%d %H:%M:%S"))
462
+ elif dtype == "STRING" or dtype == "ARRAY" or dtype == "MAP" or dtype == "STRUCT":
463
+ value_array.append(dis.read_utf().decode())
464
+ elif dtype == "INT":
465
+ value_array.append(dis.read_int())
466
+ elif dtype == "DOUBLE":
467
+ value_array.append(dis.read_double())
468
+ elif dtype == "BINARY":
469
+ value_array.append(dis.read_utf())
470
+ elif dtype == "FLOAT":
471
+ value_array.append(dis.read_float())
472
+ elif dtype == "CHAR":
473
+ value_array.append(dis.read_char())
474
+ elif dtype == "BOOLEAN":
475
+ value_array.append(dis.read_boolean())
476
+ elif dtype == "SHORT":
477
+ value_array.append(dis.read_short())
478
+ elif dtype == "BYTE":
479
+ value_array.append(dis.read_byte())
480
+ elif dtype == "INT96":
481
+ julian_day = dis.read_int()
482
+ time = dis.read_long()
483
+ date_time = datetime.fromtimestamp((julian_day - 2440588) * 86400)
484
+ date_time_with_nanos = date_time + timedelta(microseconds=(time / 1000))
485
+ value_array.append(date_time_with_nanos)
486
+ elif dtype == "INTEGER":
487
+ value_array.append(dis.read_int())
488
+ elif dtype == "DECIMAL128":
489
+ # Read decimal128 as UTF-8 string representation
490
+ decimal_str = dis.read_utf().decode()
491
+ value_array.append(Decimal(decimal_str))
492
+ except Exception as e:
493
+ _logger.error(e)
494
+ value_array.append('Failed to parse.')
495
+
496
+ return value_array
497
+
498
+
499
+ def read_rows_from_chunk(query_columns_description: list, buffer):
500
+ # Create a transport and protocol instance for deserialization
501
+ transport = TTransport.TMemoryBuffer(buffer)
502
+ protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
503
+
504
+ # Create an instance of the Thrift struct and read from the protocol
505
+ chunk = Chunk()
506
+ chunk.read(protocol)
507
+
508
+ if chunk.size <= 0:
509
+ return None
510
+
511
+ rows = list()
512
+ columns = list()
513
+
514
+ for col, colName in enumerate(query_columns_description):
515
+ columns.append(get_column_from_chunk(chunk.vectors[col]))
516
+
517
+ for rowIndex in range(chunk.size):
518
+ value = list()
519
+ for colIndex, colName in enumerate(query_columns_description):
520
+ value.append(columns[colIndex][rowIndex])
521
+ rows.append(value)
522
+
523
+ return rows
524
+
525
+
526
+ def get_column_from_chunk(vector: Vector) -> list:
527
+ value_array = list()
528
+ d_type = vector.vectorType
529
+ zone = pytz.UTC
530
+ try:
531
+ if d_type == VectorType.LONG:
532
+ for row in range(vector.size):
533
+ if get_null(vector, row):
534
+ value_array.append(None)
535
+ continue
536
+ value_array.append(vector.data.int64Data.data[
537
+ row] if not vector.isConstantVector else vector.data.numericConstantData.data)
538
+ elif d_type == VectorType.DATE:
539
+ for row in range(vector.size):
540
+ if get_null(vector, row):
541
+ value_array.append(None)
542
+ continue
543
+ epoch_seconds = floor_div(vector.data.dateData.data[
544
+ row] if not vector.isConstantVector else vector.data.dateConstantData.data,
545
+ 1000_000)
546
+ date = datetime.fromtimestamp(epoch_seconds, zone)
547
+ value_array.append(date.strftime("%Y-%m-%d"))
548
+ elif d_type == VectorType.DATETIME:
549
+ for row in range(vector.size):
550
+ if get_null(vector, row):
551
+ value_array.append(None)
552
+ continue
553
+ epoch_micros = vector.data.timeData.data[
554
+ row] if not vector.isConstantVector else vector.data.timeConstantData.data
555
+ epoch_seconds = floor_div(epoch_micros, 1000_000)
556
+ micros_of_the_day = floor_mod(epoch_micros, 1000_000)
557
+ date_time = datetime.fromtimestamp(epoch_seconds, zone)
558
+ date_time = date_time + timedelta(microseconds=micros_of_the_day)
559
+ value_array.append(date_time.isoformat(timespec='milliseconds'))
560
+ elif d_type == VectorType.STRING or d_type == VectorType.ARRAY or d_type == VectorType.MAP or d_type == VectorType.STRUCT:
561
+ for row in range(vector.size):
562
+ if get_null(vector, row):
563
+ value_array.append(None)
564
+ continue
565
+ value_array.append(vector.data.varcharData.data[
566
+ row] if not vector.isConstantVector else vector.data.varcharConstantData.data)
567
+ elif d_type == VectorType.DOUBLE:
568
+ for row in range(vector.size):
569
+ if get_null(vector, row):
570
+ value_array.append(None)
571
+ continue
572
+ value_array.append(vector.data.float64Data.data[
573
+ row] if not vector.isConstantVector else vector.data.numericDecimalConstantData.data)
574
+ elif d_type == VectorType.BINARY:
575
+ for row in range(vector.size):
576
+ if get_null(vector, row):
577
+ value_array.append(None)
578
+ continue
579
+ value_array.append(vector.data.varcharData.data[
580
+ row] if not vector.isConstantVector else vector.data.varcharConstantData.data)
581
+ elif d_type == VectorType.FLOAT:
582
+ for row in range(vector.size):
583
+ if get_null(vector, row):
584
+ value_array.append(None)
585
+ continue
586
+ value_array.append(vector.data.float32Data.data[
587
+ row] if not vector.isConstantVector else vector.data.numericDecimalConstantData.data)
588
+ elif d_type == VectorType.BOOLEAN:
589
+ for row in range(vector.size):
590
+ if get_null(vector, row):
591
+ value_array.append(None)
592
+ continue
593
+ value_array.append(vector.data.boolData.data[
594
+ row] if not vector.isConstantVector else vector.data.boolConstantData.data)
595
+ elif d_type == VectorType.INTEGER:
596
+ for row in range(vector.size):
597
+ if get_null(vector, row):
598
+ value_array.append(None)
599
+ continue
600
+ value_array.append(vector.data.int32Data.data[
601
+ row] if not vector.isConstantVector else vector.data.numericConstantData.data)
602
+ elif d_type == VectorType.NULL:
603
+ for row in range(vector.size):
604
+ value_array.append(None)
605
+ elif d_type == VectorType.TIMESTAMP_TZ:
606
+ for row in range(vector.size):
607
+ if get_null(vector, row):
608
+ value_array.append(None)
609
+ continue
610
+ epoch_micros = vector.data.timeData.data[
611
+ row] if not vector.isConstantVector else vector.data.timeConstantData.data
612
+ if ((vector.isConstantVector and vector.data.timeConstantData.zoneData is not None) or
613
+ (not vector.isConstantVector and vector.data.timeData.zoneData is not None)):
614
+ zone_id = vector.data.timeData.zoneData[
615
+ row] if not vector.isConstantVector else vector.data.timeConstantData.zoneData
616
+ zone = timezone_from_offset(zone_id)
617
+ epoch_seconds = floor_div(epoch_micros, 1000_000)
618
+ micros_of_the_day = floor_mod(epoch_micros, 1000_000)
619
+ date_time = datetime.fromtimestamp(epoch_seconds, zone)
620
+ date_time = date_time + timedelta(microseconds=micros_of_the_day)
621
+ value_array.append(date_time.isoformat(timespec='milliseconds'))
622
+ elif d_type == VectorType.DECIMAL128:
623
+ # Handle both constant and non-constant vectors following Java implementation
624
+ if vector.isConstantVector:
625
+ # For constant vectors, get the binary data and convert it once
626
+ binary_data = vector.data.numericDecimal128ConstantData.data
627
+
628
+ # Convert binary data to BigDecimal equivalent
629
+ if binary_data:
630
+ decimal_value = _binary_to_decimal128(binary_data)
631
+ else:
632
+ decimal_value = Decimal('0')
633
+
634
+ # Apply the same value to all rows
635
+ for row in range(vector.size):
636
+ if get_null(vector, row):
637
+ value_array.append(None)
638
+ else:
639
+ value_array.append(decimal_value)
640
+ else:
641
+ # For non-constant vectors, process each row individually
642
+ for row in range(vector.size):
643
+ if get_null(vector, row):
644
+ value_array.append(None)
645
+ continue
646
+ # Get binary data for this row
647
+ binary_data = vector.data.decimal128Data.data[row]
648
+ decimal_value = _binary_to_decimal128(binary_data)
649
+ value_array.append(decimal_value)
650
+ else:
651
+ value_array.append(None)
652
+ except Exception as e:
653
+ _logger.error(e)
654
+ value_array.append('Failed to parse.')
655
+ return value_array