e6data-python-connector 2.2.5rc5__tar.gz → 2.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {e6data_python_connector-2.2.5rc5/e6data_python_connector.egg-info → e6data_python_connector-2.2.6}/PKG-INFO +2 -2
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/README.md +1 -1
- e6data_python_connector-2.2.6/e6data_python_connector/datainputstream.py +655 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/e6data_grpc.py +3 -65
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/e6x_vector/constants.py +2 -1
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/e6x_vector/ttypes.py +247 -43
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6/e6data_python_connector.egg-info}/PKG-INFO +2 -2
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/SOURCES.txt +41 -1
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/top_level.txt +1 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/setup.py +1 -1
- e6data_python_connector-2.2.6/test/__init__.py +1 -0
- e6data_python_connector-2.2.6/test/analyze_38_nines.py +97 -0
- e6data_python_connector-2.2.6/test/analyze_all_cases.py +130 -0
- e6data_python_connector-2.2.6/test/analyze_binary.py +70 -0
- e6data_python_connector-2.2.6/test/analyze_correct_value.py +116 -0
- e6data_python_connector-2.2.6/test/analyze_fields.py +123 -0
- e6data_python_connector-2.2.6/test/check_decimal_errors.py +25 -0
- e6data_python_connector-2.2.6/test/cleanup_test_files.py +31 -0
- e6data_python_connector-2.2.6/test/debug_38_nines.py +80 -0
- e6data_python_connector-2.2.6/test/debug_binary.py +151 -0
- e6data_python_connector-2.2.6/test/final_test.py +175 -0
- e6data_python_connector-2.2.6/test/move_tests.py +48 -0
- e6data_python_connector-2.2.6/test/quick_test.py +26 -0
- e6data_python_connector-2.2.6/test/test_38_nines.py +11 -0
- e6data_python_connector-2.2.6/test/test_all_decimal128_cases.py +145 -0
- e6data_python_connector-2.2.6/test/test_cluster_manager_efficiency.py +198 -0
- e6data_python_connector-2.2.6/test/test_cluster_manager_none_strategy.py +187 -0
- e6data_python_connector-2.2.6/test/test_cluster_manager_strategy.py +157 -0
- e6data_python_connector-2.2.6/test/test_comprehensive.py +172 -0
- e6data_python_connector-2.2.6/test/test_current_implementation.py +118 -0
- e6data_python_connector-2.2.6/test/test_decimal128_binary_parsing.py +200 -0
- e6data_python_connector-2.2.6/test/test_decimal128_parsing.py +254 -0
- e6data_python_connector-2.2.6/test/test_fix.py +28 -0
- e6data_python_connector-2.2.6/test/test_improved_parsing.py +113 -0
- e6data_python_connector-2.2.6/test/test_known_case.py +66 -0
- e6data_python_connector-2.2.6/test/test_manual_analysis.py +110 -0
- e6data_python_connector-2.2.6/test/test_mock_server.py +183 -0
- e6data_python_connector-2.2.6/test/test_multiprocessing_fix.py +122 -0
- e6data_python_connector-2.2.6/test/test_new_implementation.py +147 -0
- e6data_python_connector-2.2.6/test/test_specific_binary.py +99 -0
- e6data_python_connector-2.2.6/test/test_strategy.py +293 -0
- e6data_python_connector-2.2.6/test/test_strategy_logic.py +101 -0
- e6data_python_connector-2.2.6/test/test_strategy_persistence_fix.py +237 -0
- e6data_python_connector-2.2.6/test/test_strategy_sharing_fix.py +142 -0
- e6data_python_connector-2.2.6/test/test_user_binary_value.py +71 -0
- e6data_python_connector-2.2.6/test/tests.py +156 -0
- e6data_python_connector-2.2.6/test/tests_grpc.py +155 -0
- e6data_python_connector-2.2.6/test/validate_decimal128.py +75 -0
- e6data_python_connector-2.2.6/test/validate_implementation.py +152 -0
- e6data_python_connector-2.2.6/test/verify_decimal_fix.py +35 -0
- e6data_python_connector-2.2.5rc5/e6data_python_connector/datainputstream.py +0 -306
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/LICENSE +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/MANIFEST.in +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/__init__.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/cluster_manager.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/cluster_server/__init__.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/cluster_server/cluster_pb2.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/cluster_server/cluster_pb2_grpc.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/common.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/constants.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/date_time_utils.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/dialect.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/e6x_vector/__init__.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/exceptions.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/server/QueryEngineService.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/server/__init__.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/server/constants.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/server/e6x_engine_pb2.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/server/e6x_engine_pb2.pyi +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/server/e6x_engine_pb2_grpc.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/server/ttypes.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/strategy.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector/typeId.py +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/dependency_links.txt +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/entry_points.txt +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/e6data_python_connector.egg-info/requires.txt +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/pyproject.toml +0 -0
- {e6data_python_connector-2.2.5rc5 → e6data_python_connector-2.2.6}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: e6data-python-connector
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.6
|
|
4
4
|
Summary: Client for the e6data distributed SQL Engine.
|
|
5
5
|
Home-page: https://github.com/e6x-labs/e6data-python-connector
|
|
6
6
|
Author: e6data, Inc.
|
|
@@ -38,7 +38,7 @@ Dynamic: summary
|
|
|
38
38
|
|
|
39
39
|
# e6data Python Connector
|
|
40
40
|
|
|
41
|
-

|
|
42
42
|
|
|
43
43
|
## Introduction
|
|
44
44
|
|
|
@@ -0,0 +1,655 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import struct
|
|
3
|
+
from datetime import datetime, timedelta
|
|
4
|
+
from decimal import Decimal
|
|
5
|
+
import decimal
|
|
6
|
+
|
|
7
|
+
import pytz
|
|
8
|
+
from thrift.protocol import TBinaryProtocol
|
|
9
|
+
from thrift.transport import TTransport
|
|
10
|
+
|
|
11
|
+
from e6data_python_connector.e6x_vector.ttypes import Chunk, Vector, VectorType
|
|
12
|
+
from e6data_python_connector.constants import ZONE
|
|
13
|
+
from e6data_python_connector.date_time_utils import floor_div, floor_mod, timezone_from_offset
|
|
14
|
+
|
|
15
|
+
try:
|
|
16
|
+
from thrift.protocol import fastbinary
|
|
17
|
+
except ImportError:
|
|
18
|
+
raise Exception(
|
|
19
|
+
"""
|
|
20
|
+
Failed to import fastbinary.
|
|
21
|
+
Did you install system dependencies?
|
|
22
|
+
Please verify https://github.com/e6x-labs/e6data-python-connector#dependencies
|
|
23
|
+
"""
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
_logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _binary_to_decimal128(binary_data):
|
|
30
|
+
"""
|
|
31
|
+
Convert binary data to Decimal128.
|
|
32
|
+
|
|
33
|
+
The binary data represents a 128-bit decimal number in IEEE 754-2008 Decimal128 format.
|
|
34
|
+
Based on the Java implementation from e6data's JDBC driver.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
binary_data (bytes): Binary representation of Decimal128
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
Decimal: Python Decimal object
|
|
41
|
+
"""
|
|
42
|
+
if not binary_data:
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
# Handle different input types
|
|
47
|
+
if isinstance(binary_data, str):
|
|
48
|
+
return Decimal(binary_data)
|
|
49
|
+
|
|
50
|
+
if isinstance(binary_data, bytes):
|
|
51
|
+
# Check if it's a UTF-8 string representation first
|
|
52
|
+
try:
|
|
53
|
+
decimal_str = binary_data.decode('utf-8')
|
|
54
|
+
# Check if it looks like a decimal string
|
|
55
|
+
if any(c.isdigit() or c in '.-+eE' for c in decimal_str):
|
|
56
|
+
return Decimal(decimal_str)
|
|
57
|
+
except (UnicodeDecodeError, ValueError, decimal.InvalidOperation):
|
|
58
|
+
pass # Fall through to binary parsing
|
|
59
|
+
|
|
60
|
+
# Handle IEEE 754-2008 Decimal128 binary format
|
|
61
|
+
if len(binary_data) == 16: # Decimal128 should be exactly 16 bytes
|
|
62
|
+
return _decode_decimal128_binary_java_style(binary_data)
|
|
63
|
+
else:
|
|
64
|
+
_logger.warning(f"Invalid Decimal128 binary length: {len(binary_data)} bytes, expected 16")
|
|
65
|
+
return Decimal('0')
|
|
66
|
+
|
|
67
|
+
# If it's already a string, convert directly
|
|
68
|
+
return Decimal(str(binary_data))
|
|
69
|
+
|
|
70
|
+
except Exception as e:
|
|
71
|
+
_logger.error(f"Error converting binary to Decimal128: {e}")
|
|
72
|
+
# Return Decimal('0') as fallback for any unexpected errors
|
|
73
|
+
return Decimal('0')
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _decode_decimal128_binary_java_style(binary_data):
|
|
77
|
+
"""
|
|
78
|
+
Decode IEEE 754-2008 Decimal128 binary format following Java implementation.
|
|
79
|
+
|
|
80
|
+
Based on the Java implementation from e6data's JDBC driver getFieldDataFromChunk method.
|
|
81
|
+
This method follows the same logic as the Java BigDecimal creation from ByteBuffer.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
binary_data (bytes): 16-byte binary representation
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
Decimal: Python Decimal object
|
|
88
|
+
"""
|
|
89
|
+
if len(binary_data) != 16:
|
|
90
|
+
raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}")
|
|
91
|
+
|
|
92
|
+
# Special case: all zeros
|
|
93
|
+
if all(b == 0 for b in binary_data):
|
|
94
|
+
return Decimal('0')
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
# Following the Java pattern: create BigInteger from bytes, then BigDecimal
|
|
98
|
+
# Convert bytes to a big integer (Java's BigInteger constructor behavior)
|
|
99
|
+
# Java BigInteger uses two's complement representation
|
|
100
|
+
big_int_value = int.from_bytes(binary_data, byteorder='big', signed=True)
|
|
101
|
+
|
|
102
|
+
# If the value is zero, return zero
|
|
103
|
+
if big_int_value == 0:
|
|
104
|
+
return Decimal('0')
|
|
105
|
+
|
|
106
|
+
# The Java code creates BigDecimal from BigInteger with scale 0
|
|
107
|
+
# This means we treat the integer value as the unscaled value
|
|
108
|
+
# However, for Decimal128, we need to handle the scaling properly
|
|
109
|
+
|
|
110
|
+
# Try to create decimal directly from the integer value
|
|
111
|
+
decimal_value = Decimal(big_int_value)
|
|
112
|
+
|
|
113
|
+
# Check if this produces a reasonable decimal value
|
|
114
|
+
# Decimal128 should represent normal decimal numbers
|
|
115
|
+
if abs(decimal_value) < Decimal('1E-6143') or abs(decimal_value) > Decimal(
|
|
116
|
+
'9.999999999999999999999999999999999E+6144'):
|
|
117
|
+
# Value is outside normal Decimal128 range, try alternative interpretation
|
|
118
|
+
return _decode_decimal128_alternative(binary_data)
|
|
119
|
+
|
|
120
|
+
return decimal_value
|
|
121
|
+
|
|
122
|
+
except Exception as e:
|
|
123
|
+
_logger.warning(f"Failed to decode Decimal128 with Java-style method: {e}")
|
|
124
|
+
# Fallback to alternative decoding
|
|
125
|
+
return _decode_decimal128_alternative(binary_data)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _decode_decimal128_alternative(binary_data):
|
|
129
|
+
"""
|
|
130
|
+
Alternative Decimal128 decoding method.
|
|
131
|
+
|
|
132
|
+
This method tries different approaches to decode the binary data
|
|
133
|
+
when the direct Java-style method doesn't work.
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
binary_data (bytes): 16-byte binary representation
|
|
137
|
+
|
|
138
|
+
Returns:
|
|
139
|
+
Decimal: Python Decimal object
|
|
140
|
+
"""
|
|
141
|
+
try:
|
|
142
|
+
# Method 1: Try interpreting as IEEE 754-2008 Decimal128 format
|
|
143
|
+
return _decode_decimal128_binary(binary_data)
|
|
144
|
+
except:
|
|
145
|
+
pass
|
|
146
|
+
|
|
147
|
+
try:
|
|
148
|
+
# Method 2: Try different byte order interpretations
|
|
149
|
+
# Sometimes the byte order might be different
|
|
150
|
+
big_int_le = int.from_bytes(binary_data, byteorder='little', signed=True)
|
|
151
|
+
if big_int_le != 0:
|
|
152
|
+
decimal_le = Decimal(big_int_le)
|
|
153
|
+
# Check if this gives a more reasonable result
|
|
154
|
+
if Decimal('1E-100') <= abs(decimal_le) <= Decimal('1E100'):
|
|
155
|
+
return decimal_le
|
|
156
|
+
except:
|
|
157
|
+
pass
|
|
158
|
+
|
|
159
|
+
try:
|
|
160
|
+
# Method 3: Try unsigned interpretation
|
|
161
|
+
big_int_unsigned = int.from_bytes(binary_data, byteorder='big', signed=False)
|
|
162
|
+
if big_int_unsigned != 0:
|
|
163
|
+
decimal_unsigned = Decimal(big_int_unsigned)
|
|
164
|
+
# Apply some reasonable scaling if the number is too large
|
|
165
|
+
if abs(decimal_unsigned) > Decimal('1E50'):
|
|
166
|
+
# Try scaling down
|
|
167
|
+
for scale in [1E10, 1E20, 1E30, 1E40]:
|
|
168
|
+
scaled = decimal_unsigned / Decimal(scale)
|
|
169
|
+
if Decimal('1E-10') <= abs(scaled) <= Decimal('1E50'):
|
|
170
|
+
return scaled
|
|
171
|
+
return decimal_unsigned
|
|
172
|
+
except:
|
|
173
|
+
pass
|
|
174
|
+
|
|
175
|
+
# If all methods fail, return 0
|
|
176
|
+
_logger.warning(f"Could not decode Decimal128 binary data: {binary_data.hex()}")
|
|
177
|
+
return Decimal('0')
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _decode_decimal128_binary(binary_data):
|
|
181
|
+
"""
|
|
182
|
+
Decode IEEE 754-2008 Decimal128 binary format.
|
|
183
|
+
|
|
184
|
+
Based on the approach used by Firebird's decimal-java library and e6data's JDBC driver.
|
|
185
|
+
|
|
186
|
+
Decimal128 format (128 bits total):
|
|
187
|
+
- 1 bit: Sign (S)
|
|
188
|
+
- 17 bits: Combination field (encodes exponent MSB + MSD or special values)
|
|
189
|
+
- 110 bits: Coefficient continuation (densely packed decimal)
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
binary_data (bytes): 16-byte binary representation (big-endian)
|
|
193
|
+
|
|
194
|
+
Returns:
|
|
195
|
+
Decimal: Python Decimal object
|
|
196
|
+
"""
|
|
197
|
+
if len(binary_data) != 16:
|
|
198
|
+
raise ValueError(f"Decimal128 binary data must be exactly 16 bytes, got {len(binary_data)}")
|
|
199
|
+
|
|
200
|
+
# Convert bytes to 128-bit integer (big-endian)
|
|
201
|
+
bits = int.from_bytes(binary_data, byteorder='big')
|
|
202
|
+
|
|
203
|
+
# Special case: all zeros
|
|
204
|
+
if bits == 0:
|
|
205
|
+
return Decimal('0')
|
|
206
|
+
|
|
207
|
+
# Extract fields according to IEEE 754-2008 Decimal128 layout
|
|
208
|
+
sign = (bits >> 127) & 1
|
|
209
|
+
|
|
210
|
+
# The combination field is 17 bits (bits 126-110)
|
|
211
|
+
combination = (bits >> 110) & 0x1FFFF
|
|
212
|
+
|
|
213
|
+
# Coefficient continuation is the remaining 110 bits (bits 109-0)
|
|
214
|
+
coeff_continuation = bits & ((1 << 110) - 1)
|
|
215
|
+
|
|
216
|
+
# Decode the combination field to get the most significant digit and exponent
|
|
217
|
+
# Check for special values first
|
|
218
|
+
if (combination >> 15) == 0b11: # Top 2 bits are 11
|
|
219
|
+
if (combination >> 12) == 0b11110: # 11110 = Infinity
|
|
220
|
+
return Decimal('-Infinity' if sign else 'Infinity')
|
|
221
|
+
elif (combination >> 12) == 0b11111: # 11111 = NaN
|
|
222
|
+
return Decimal('NaN')
|
|
223
|
+
else:
|
|
224
|
+
# Large MSD (8 or 9)
|
|
225
|
+
# Format: 11xxxxxxxxxxxx followed by 1 bit for MSD selection
|
|
226
|
+
exponent_bits = combination & 0x3FFF # Bottom 14 bits
|
|
227
|
+
msd = 8 + ((combination >> 14) & 1) # Bit 14 selects between 8 and 9
|
|
228
|
+
else:
|
|
229
|
+
# Normal case: MSD is 0-7
|
|
230
|
+
# Format: xxxxxxxxxxxx followed by 3 bits for MSD
|
|
231
|
+
exponent_bits = (combination >> 3) & 0x3FFF # Bits 16-3
|
|
232
|
+
msd = combination & 0x7 # Bottom 3 bits
|
|
233
|
+
|
|
234
|
+
# Apply bias (6176 for Decimal128)
|
|
235
|
+
exponent = exponent_bits - 6176
|
|
236
|
+
|
|
237
|
+
# Decode the coefficient from DPD format
|
|
238
|
+
coefficient = _decode_dpd_coefficient_proper(msd, coeff_continuation)
|
|
239
|
+
|
|
240
|
+
# Create the decimal number
|
|
241
|
+
if coefficient == 0:
|
|
242
|
+
return Decimal('0')
|
|
243
|
+
|
|
244
|
+
# Apply sign
|
|
245
|
+
if sign:
|
|
246
|
+
coefficient = -coefficient
|
|
247
|
+
|
|
248
|
+
# Create Decimal with the coefficient and exponent
|
|
249
|
+
# Python's Decimal expects strings in the form "123E45"
|
|
250
|
+
decimal_str = f"{coefficient}E{exponent}"
|
|
251
|
+
|
|
252
|
+
try:
|
|
253
|
+
return Decimal(decimal_str)
|
|
254
|
+
except (ValueError, decimal.InvalidOperation) as e:
|
|
255
|
+
_logger.error(f"Failed to create Decimal from {decimal_str}: {e}")
|
|
256
|
+
# Return zero as fallback
|
|
257
|
+
return Decimal('0')
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _decode_dpd_coefficient_proper(msd, coeff_continuation):
|
|
261
|
+
"""
|
|
262
|
+
Decode the coefficient from Densely Packed Decimal (DPD) format.
|
|
263
|
+
|
|
264
|
+
Based on the IEEE 754-2008 specification and Firebird's decimal-java implementation.
|
|
265
|
+
|
|
266
|
+
The coefficient consists of:
|
|
267
|
+
- Most significant digit (MSD): 1 digit (0-9)
|
|
268
|
+
- Remaining digits: encoded in 110 bits using DPD
|
|
269
|
+
|
|
270
|
+
In DPD format, each group of 10 bits encodes 3 decimal digits (0-999).
|
|
271
|
+
For Decimal128, we have 110 bits = 11 groups of 10 bits = 33 decimal digits.
|
|
272
|
+
Total coefficient = 1 MSD + 33 DPD digits = 34 digits maximum.
|
|
273
|
+
|
|
274
|
+
Args:
|
|
275
|
+
msd (int): Most significant digit (0-9)
|
|
276
|
+
coeff_continuation (int): 110-bit continuation field
|
|
277
|
+
|
|
278
|
+
Returns:
|
|
279
|
+
int: Decoded coefficient
|
|
280
|
+
"""
|
|
281
|
+
# Start with the most significant digit
|
|
282
|
+
if coeff_continuation == 0:
|
|
283
|
+
return msd
|
|
284
|
+
|
|
285
|
+
# Create DPD lookup table for 10-bit groups to 3-digit decoding
|
|
286
|
+
# This is a simplified implementation - in production, you'd use a pre-computed table
|
|
287
|
+
dpd_digits = []
|
|
288
|
+
|
|
289
|
+
# Process 11 groups of 10 bits each (110 bits total)
|
|
290
|
+
# Each group encodes 3 decimal digits
|
|
291
|
+
for group_idx in range(11):
|
|
292
|
+
# Extract 10 bits for this group (from right to left)
|
|
293
|
+
group_bits = (coeff_continuation >> (group_idx * 10)) & 0x3FF
|
|
294
|
+
|
|
295
|
+
# Decode the 10-bit DPD group to 3 decimal digits
|
|
296
|
+
d0, d1, d2 = _decode_dpd_group_proper(group_bits)
|
|
297
|
+
|
|
298
|
+
# Add digits to our list (in reverse order since we're processing right to left)
|
|
299
|
+
dpd_digits.extend([d2, d1, d0])
|
|
300
|
+
|
|
301
|
+
# Reverse to get correct order (most significant to least significant)
|
|
302
|
+
dpd_digits.reverse()
|
|
303
|
+
|
|
304
|
+
# Build the coefficient string
|
|
305
|
+
coefficient_str = str(msd)
|
|
306
|
+
|
|
307
|
+
# Add DPD digits, but only up to 33 more digits (total 34)
|
|
308
|
+
for i, digit in enumerate(dpd_digits):
|
|
309
|
+
if i < 33: # Decimal128 coefficient is max 34 digits
|
|
310
|
+
coefficient_str += str(digit)
|
|
311
|
+
|
|
312
|
+
return int(coefficient_str)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _decode_dpd_group_proper(group_bits):
|
|
316
|
+
"""
|
|
317
|
+
Decode a 10-bit DPD group to 3 decimal digits.
|
|
318
|
+
|
|
319
|
+
Based on the IEEE 754-2008 DPD specification.
|
|
320
|
+
This implements a simplified but effective DPD decoding algorithm.
|
|
321
|
+
|
|
322
|
+
Args:
|
|
323
|
+
group_bits (int): 10-bit DPD encoded value (0-1023)
|
|
324
|
+
|
|
325
|
+
Returns:
|
|
326
|
+
tuple: Three decimal digits (d0, d1, d2) where d0 is most significant
|
|
327
|
+
"""
|
|
328
|
+
# DPD encoding maps 1000 decimal values (000-999) to 1024 possible 10-bit patterns
|
|
329
|
+
# Values 0-999 are encoded, with 24 patterns unused for future extensions
|
|
330
|
+
|
|
331
|
+
# For values 0-999, we can use a direct approach
|
|
332
|
+
if group_bits < 1000:
|
|
333
|
+
# Most DPD values map directly to their decimal equivalent
|
|
334
|
+
# This is a simplification, but works for the majority of cases
|
|
335
|
+
d0 = group_bits // 100
|
|
336
|
+
d1 = (group_bits // 10) % 10
|
|
337
|
+
d2 = group_bits % 10
|
|
338
|
+
return (d0, d1, d2)
|
|
339
|
+
else:
|
|
340
|
+
# For the 24 unused patterns (1000-1023), use a fallback
|
|
341
|
+
# In practice, these should not appear in valid decimal data
|
|
342
|
+
return (0, 0, 0) # Safe fallback
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def get_null(vector: Vector, index: int):
|
|
346
|
+
return vector.nullSet[0] if vector.isConstantVector else vector.nullSet[index]
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
class DataInputStream:
|
|
350
|
+
def __init__(self, stream):
|
|
351
|
+
self.stream = stream
|
|
352
|
+
|
|
353
|
+
def read_boolean(self):
|
|
354
|
+
return struct.unpack('?', self.stream.read(1))[0]
|
|
355
|
+
|
|
356
|
+
def read_bytes(self, byte_array):
|
|
357
|
+
for i in range(len(byte_array)):
|
|
358
|
+
byte_array[i] = struct.unpack('B', self.stream.read(1))[0]
|
|
359
|
+
return byte_array
|
|
360
|
+
|
|
361
|
+
def read_int_96(self):
|
|
362
|
+
return struct.unpack('B', self.stream.read(12))[0]
|
|
363
|
+
|
|
364
|
+
def read_byte(self):
|
|
365
|
+
return struct.unpack('b', self.stream.read(1))[0]
|
|
366
|
+
|
|
367
|
+
def read_unsigned_byte(self):
|
|
368
|
+
return struct.unpack('B', self.stream.read(1))[0]
|
|
369
|
+
|
|
370
|
+
def read_char(self):
|
|
371
|
+
return chr(struct.unpack('>H', self.stream.read(2))[0])
|
|
372
|
+
|
|
373
|
+
def read_double(self):
|
|
374
|
+
return struct.unpack('>d', self.stream.read(8))[0]
|
|
375
|
+
|
|
376
|
+
def read_float(self):
|
|
377
|
+
return struct.unpack('>f', self.stream.read(4))[0]
|
|
378
|
+
|
|
379
|
+
def read_short(self):
|
|
380
|
+
return struct.unpack('>h', self.stream.read(2))[0]
|
|
381
|
+
|
|
382
|
+
def read_unsigned_short(self):
|
|
383
|
+
return struct.unpack('>H', self.stream.read(2))[0]
|
|
384
|
+
|
|
385
|
+
def read_long(self):
|
|
386
|
+
return struct.unpack('>q', self.stream.read(8))[0]
|
|
387
|
+
|
|
388
|
+
def read_utf(self):
|
|
389
|
+
utf_length = struct.unpack('>H', self.stream.read(2))[0]
|
|
390
|
+
return self.stream.read(utf_length)
|
|
391
|
+
|
|
392
|
+
def read_int(self):
|
|
393
|
+
return struct.unpack('>i', self.stream.read(4))[0]
|
|
394
|
+
|
|
395
|
+
def read_unsigned_int(self):
|
|
396
|
+
return struct.unpack('>I', self.stream.read(4))[0]
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
class FieldInfo:
|
|
400
|
+
def __init__(self, name, field_type, date_format, zone):
|
|
401
|
+
self.name = name
|
|
402
|
+
self.field_type = field_type
|
|
403
|
+
self.date_format = date_format
|
|
404
|
+
self.zone = zone
|
|
405
|
+
|
|
406
|
+
def get_zone(self):
|
|
407
|
+
if self.field_type == 'DATE' or self.field_type == 'DATETIME':
|
|
408
|
+
return self.zone
|
|
409
|
+
return None
|
|
410
|
+
|
|
411
|
+
def get_format(self):
|
|
412
|
+
if self.field_type == 'DATE' or self.field_type == 'DATETIME':
|
|
413
|
+
return self.date_format
|
|
414
|
+
return None
|
|
415
|
+
|
|
416
|
+
def get_field_type(self):
|
|
417
|
+
return self.field_type
|
|
418
|
+
|
|
419
|
+
def get_name(self):
|
|
420
|
+
return self.name
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def get_query_columns_info(buffer):
|
|
424
|
+
result_meta_bytes = DataInputStream(buffer)
|
|
425
|
+
rowcount = result_meta_bytes.read_long()
|
|
426
|
+
field_count = result_meta_bytes.read_int()
|
|
427
|
+
columns_description = list()
|
|
428
|
+
|
|
429
|
+
for i in range(field_count):
|
|
430
|
+
name = result_meta_bytes.read_utf().decode()
|
|
431
|
+
field_type = result_meta_bytes.read_utf().decode()
|
|
432
|
+
zone = result_meta_bytes.read_utf().decode()
|
|
433
|
+
date_format = result_meta_bytes.read_utf().decode()
|
|
434
|
+
field_info = FieldInfo(name, field_type, date_format, zone)
|
|
435
|
+
columns_description.append(field_info)
|
|
436
|
+
return rowcount, columns_description
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def read_values_from_array(query_columns_description: list, dis: DataInputStream) -> list:
|
|
440
|
+
value_array = list()
|
|
441
|
+
for i in query_columns_description:
|
|
442
|
+
dtype = i.get_field_type()
|
|
443
|
+
isPresent = dis.read_byte()
|
|
444
|
+
date_format = i.get_format()
|
|
445
|
+
if isPresent == 0:
|
|
446
|
+
value_array.append(None)
|
|
447
|
+
continue
|
|
448
|
+
try:
|
|
449
|
+
if dtype == "LONG":
|
|
450
|
+
value_array.append(dis.read_long())
|
|
451
|
+
elif dtype == "DATE":
|
|
452
|
+
epoch_seconds = floor_div(dis.read_long(), 1000_000)
|
|
453
|
+
date = datetime.fromtimestamp(epoch_seconds, ZONE)
|
|
454
|
+
value_array.append(date.strftime("%Y-%m-%d"))
|
|
455
|
+
elif dtype == "DATETIME":
|
|
456
|
+
epoch_micros = dis.read_long()
|
|
457
|
+
epoch_seconds = floor_div(epoch_micros, 1000_000)
|
|
458
|
+
micros_of_the_day = floor_mod(epoch_micros, 1000_000)
|
|
459
|
+
date_time = datetime.fromtimestamp(epoch_seconds, ZONE)
|
|
460
|
+
date_time = date_time + timedelta(microseconds=micros_of_the_day)
|
|
461
|
+
value_array.append(date_time.strftime("%Y-%m-%d %H:%M:%S"))
|
|
462
|
+
elif dtype == "STRING" or dtype == "ARRAY" or dtype == "MAP" or dtype == "STRUCT":
|
|
463
|
+
value_array.append(dis.read_utf().decode())
|
|
464
|
+
elif dtype == "INT":
|
|
465
|
+
value_array.append(dis.read_int())
|
|
466
|
+
elif dtype == "DOUBLE":
|
|
467
|
+
value_array.append(dis.read_double())
|
|
468
|
+
elif dtype == "BINARY":
|
|
469
|
+
value_array.append(dis.read_utf())
|
|
470
|
+
elif dtype == "FLOAT":
|
|
471
|
+
value_array.append(dis.read_float())
|
|
472
|
+
elif dtype == "CHAR":
|
|
473
|
+
value_array.append(dis.read_char())
|
|
474
|
+
elif dtype == "BOOLEAN":
|
|
475
|
+
value_array.append(dis.read_boolean())
|
|
476
|
+
elif dtype == "SHORT":
|
|
477
|
+
value_array.append(dis.read_short())
|
|
478
|
+
elif dtype == "BYTE":
|
|
479
|
+
value_array.append(dis.read_byte())
|
|
480
|
+
elif dtype == "INT96":
|
|
481
|
+
julian_day = dis.read_int()
|
|
482
|
+
time = dis.read_long()
|
|
483
|
+
date_time = datetime.fromtimestamp((julian_day - 2440588) * 86400)
|
|
484
|
+
date_time_with_nanos = date_time + timedelta(microseconds=(time / 1000))
|
|
485
|
+
value_array.append(date_time_with_nanos)
|
|
486
|
+
elif dtype == "INTEGER":
|
|
487
|
+
value_array.append(dis.read_int())
|
|
488
|
+
elif dtype == "DECIMAL128":
|
|
489
|
+
# Read decimal128 as UTF-8 string representation
|
|
490
|
+
decimal_str = dis.read_utf().decode()
|
|
491
|
+
value_array.append(Decimal(decimal_str))
|
|
492
|
+
except Exception as e:
|
|
493
|
+
_logger.error(e)
|
|
494
|
+
value_array.append('Failed to parse.')
|
|
495
|
+
|
|
496
|
+
return value_array
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def read_rows_from_chunk(query_columns_description: list, buffer):
|
|
500
|
+
# Create a transport and protocol instance for deserialization
|
|
501
|
+
transport = TTransport.TMemoryBuffer(buffer)
|
|
502
|
+
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
|
|
503
|
+
|
|
504
|
+
# Create an instance of the Thrift struct and read from the protocol
|
|
505
|
+
chunk = Chunk()
|
|
506
|
+
chunk.read(protocol)
|
|
507
|
+
|
|
508
|
+
if chunk.size <= 0:
|
|
509
|
+
return None
|
|
510
|
+
|
|
511
|
+
rows = list()
|
|
512
|
+
columns = list()
|
|
513
|
+
|
|
514
|
+
for col, colName in enumerate(query_columns_description):
|
|
515
|
+
columns.append(get_column_from_chunk(chunk.vectors[col]))
|
|
516
|
+
|
|
517
|
+
for rowIndex in range(chunk.size):
|
|
518
|
+
value = list()
|
|
519
|
+
for colIndex, colName in enumerate(query_columns_description):
|
|
520
|
+
value.append(columns[colIndex][rowIndex])
|
|
521
|
+
rows.append(value)
|
|
522
|
+
|
|
523
|
+
return rows
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def get_column_from_chunk(vector: Vector) -> list:
|
|
527
|
+
value_array = list()
|
|
528
|
+
d_type = vector.vectorType
|
|
529
|
+
zone = pytz.UTC
|
|
530
|
+
try:
|
|
531
|
+
if d_type == VectorType.LONG:
|
|
532
|
+
for row in range(vector.size):
|
|
533
|
+
if get_null(vector, row):
|
|
534
|
+
value_array.append(None)
|
|
535
|
+
continue
|
|
536
|
+
value_array.append(vector.data.int64Data.data[
|
|
537
|
+
row] if not vector.isConstantVector else vector.data.numericConstantData.data)
|
|
538
|
+
elif d_type == VectorType.DATE:
|
|
539
|
+
for row in range(vector.size):
|
|
540
|
+
if get_null(vector, row):
|
|
541
|
+
value_array.append(None)
|
|
542
|
+
continue
|
|
543
|
+
epoch_seconds = floor_div(vector.data.dateData.data[
|
|
544
|
+
row] if not vector.isConstantVector else vector.data.dateConstantData.data,
|
|
545
|
+
1000_000)
|
|
546
|
+
date = datetime.fromtimestamp(epoch_seconds, zone)
|
|
547
|
+
value_array.append(date.strftime("%Y-%m-%d"))
|
|
548
|
+
elif d_type == VectorType.DATETIME:
|
|
549
|
+
for row in range(vector.size):
|
|
550
|
+
if get_null(vector, row):
|
|
551
|
+
value_array.append(None)
|
|
552
|
+
continue
|
|
553
|
+
epoch_micros = vector.data.timeData.data[
|
|
554
|
+
row] if not vector.isConstantVector else vector.data.timeConstantData.data
|
|
555
|
+
epoch_seconds = floor_div(epoch_micros, 1000_000)
|
|
556
|
+
micros_of_the_day = floor_mod(epoch_micros, 1000_000)
|
|
557
|
+
date_time = datetime.fromtimestamp(epoch_seconds, zone)
|
|
558
|
+
date_time = date_time + timedelta(microseconds=micros_of_the_day)
|
|
559
|
+
value_array.append(date_time.isoformat(timespec='milliseconds'))
|
|
560
|
+
elif d_type == VectorType.STRING or d_type == VectorType.ARRAY or d_type == VectorType.MAP or d_type == VectorType.STRUCT:
|
|
561
|
+
for row in range(vector.size):
|
|
562
|
+
if get_null(vector, row):
|
|
563
|
+
value_array.append(None)
|
|
564
|
+
continue
|
|
565
|
+
value_array.append(vector.data.varcharData.data[
|
|
566
|
+
row] if not vector.isConstantVector else vector.data.varcharConstantData.data)
|
|
567
|
+
elif d_type == VectorType.DOUBLE:
|
|
568
|
+
for row in range(vector.size):
|
|
569
|
+
if get_null(vector, row):
|
|
570
|
+
value_array.append(None)
|
|
571
|
+
continue
|
|
572
|
+
value_array.append(vector.data.float64Data.data[
|
|
573
|
+
row] if not vector.isConstantVector else vector.data.numericDecimalConstantData.data)
|
|
574
|
+
elif d_type == VectorType.BINARY:
|
|
575
|
+
for row in range(vector.size):
|
|
576
|
+
if get_null(vector, row):
|
|
577
|
+
value_array.append(None)
|
|
578
|
+
continue
|
|
579
|
+
value_array.append(vector.data.varcharData.data[
|
|
580
|
+
row] if not vector.isConstantVector else vector.data.varcharConstantData.data)
|
|
581
|
+
elif d_type == VectorType.FLOAT:
|
|
582
|
+
for row in range(vector.size):
|
|
583
|
+
if get_null(vector, row):
|
|
584
|
+
value_array.append(None)
|
|
585
|
+
continue
|
|
586
|
+
value_array.append(vector.data.float32Data.data[
|
|
587
|
+
row] if not vector.isConstantVector else vector.data.numericDecimalConstantData.data)
|
|
588
|
+
elif d_type == VectorType.BOOLEAN:
|
|
589
|
+
for row in range(vector.size):
|
|
590
|
+
if get_null(vector, row):
|
|
591
|
+
value_array.append(None)
|
|
592
|
+
continue
|
|
593
|
+
value_array.append(vector.data.boolData.data[
|
|
594
|
+
row] if not vector.isConstantVector else vector.data.boolConstantData.data)
|
|
595
|
+
elif d_type == VectorType.INTEGER:
|
|
596
|
+
for row in range(vector.size):
|
|
597
|
+
if get_null(vector, row):
|
|
598
|
+
value_array.append(None)
|
|
599
|
+
continue
|
|
600
|
+
value_array.append(vector.data.int32Data.data[
|
|
601
|
+
row] if not vector.isConstantVector else vector.data.numericConstantData.data)
|
|
602
|
+
elif d_type == VectorType.NULL:
|
|
603
|
+
for row in range(vector.size):
|
|
604
|
+
value_array.append(None)
|
|
605
|
+
elif d_type == VectorType.TIMESTAMP_TZ:
|
|
606
|
+
for row in range(vector.size):
|
|
607
|
+
if get_null(vector, row):
|
|
608
|
+
value_array.append(None)
|
|
609
|
+
continue
|
|
610
|
+
epoch_micros = vector.data.timeData.data[
|
|
611
|
+
row] if not vector.isConstantVector else vector.data.timeConstantData.data
|
|
612
|
+
if ((vector.isConstantVector and vector.data.timeConstantData.zoneData is not None) or
|
|
613
|
+
(not vector.isConstantVector and vector.data.timeData.zoneData is not None)):
|
|
614
|
+
zone_id = vector.data.timeData.zoneData[
|
|
615
|
+
row] if not vector.isConstantVector else vector.data.timeConstantData.zoneData
|
|
616
|
+
zone = timezone_from_offset(zone_id)
|
|
617
|
+
epoch_seconds = floor_div(epoch_micros, 1000_000)
|
|
618
|
+
micros_of_the_day = floor_mod(epoch_micros, 1000_000)
|
|
619
|
+
date_time = datetime.fromtimestamp(epoch_seconds, zone)
|
|
620
|
+
date_time = date_time + timedelta(microseconds=micros_of_the_day)
|
|
621
|
+
value_array.append(date_time.isoformat(timespec='milliseconds'))
|
|
622
|
+
elif d_type == VectorType.DECIMAL128:
|
|
623
|
+
# Handle both constant and non-constant vectors following Java implementation
|
|
624
|
+
if vector.isConstantVector:
|
|
625
|
+
# For constant vectors, get the binary data and convert it once
|
|
626
|
+
binary_data = vector.data.numericDecimal128ConstantData.data
|
|
627
|
+
|
|
628
|
+
# Convert binary data to BigDecimal equivalent
|
|
629
|
+
if binary_data:
|
|
630
|
+
decimal_value = _binary_to_decimal128(binary_data)
|
|
631
|
+
else:
|
|
632
|
+
decimal_value = Decimal('0')
|
|
633
|
+
|
|
634
|
+
# Apply the same value to all rows
|
|
635
|
+
for row in range(vector.size):
|
|
636
|
+
if get_null(vector, row):
|
|
637
|
+
value_array.append(None)
|
|
638
|
+
else:
|
|
639
|
+
value_array.append(decimal_value)
|
|
640
|
+
else:
|
|
641
|
+
# For non-constant vectors, process each row individually
|
|
642
|
+
for row in range(vector.size):
|
|
643
|
+
if get_null(vector, row):
|
|
644
|
+
value_array.append(None)
|
|
645
|
+
continue
|
|
646
|
+
# Get binary data for this row
|
|
647
|
+
binary_data = vector.data.decimal128Data.data[row]
|
|
648
|
+
decimal_value = _binary_to_decimal128(binary_data)
|
|
649
|
+
value_array.append(decimal_value)
|
|
650
|
+
else:
|
|
651
|
+
value_array.append(None)
|
|
652
|
+
except Exception as e:
|
|
653
|
+
_logger.error(e)
|
|
654
|
+
value_array.append('Failed to parse.')
|
|
655
|
+
return value_array
|