exceeddata-sdk-vdata 2.9.3.2__tar.gz → 2.9.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exceeddata-sdk-vdata-2.9.6.1/LICENSE +29 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/PKG-INFO +1 -1
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata/sdk/__init__.py +1 -1
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata/sdk/vdata.py +287 -33
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata_sdk_vdata.egg-info/PKG-INFO +1 -1
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata_sdk_vdata.egg-info/SOURCES.txt +1 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/setup.py +1 -1
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/README.rst +0 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata/sdk/blf.py +0 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata/sdk/converter.py +0 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata_sdk_vdata.egg-info/dependency_links.txt +0 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata_sdk_vdata.egg-info/not-zip-safe +0 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata_sdk_vdata.egg-info/requires.txt +0 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/exceeddata_sdk_vdata.egg-info/top_level.txt +0 -0
- {exceeddata-sdk-vdata-2.9.3.2 → exceeddata-sdk-vdata-2.9.6.1}/setup.cfg +0 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Copyright (C) 2016-2024 Smart Software for Car Technologies Inc. and EXCEEDDATA
|
|
2
|
+
https://www.smartsct.com
|
|
3
|
+
https://www.exceeddata.com
|
|
4
|
+
|
|
5
|
+
MIT License
|
|
6
|
+
|
|
7
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
8
|
+
a copy of this software and associated documentation files (the
|
|
9
|
+
"Software"), to deal in the Software without restriction, including
|
|
10
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
11
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
12
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
13
|
+
the following conditions:
|
|
14
|
+
|
|
15
|
+
The above copyright notice and this permission notice shall be
|
|
16
|
+
included in all copies or substantial portions of the Software.
|
|
17
|
+
|
|
18
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
19
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
20
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
21
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
|
22
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
|
23
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
|
24
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
25
|
+
|
|
26
|
+
Except as contained in this notice, the name of a copyright holder
|
|
27
|
+
shall not be used in advertising or otherwise to promote the sale, use
|
|
28
|
+
or other dealings in this Software without prior written authorization
|
|
29
|
+
of the copyright holder.
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
# of the copyright holder.
|
|
32
32
|
#
|
|
33
33
|
# Author: Nick Xie
|
|
34
|
-
# Version: 2.9.
|
|
34
|
+
# Version: 2.9.6.1
|
|
35
35
|
#
|
|
36
36
|
# Note: for specification flag compatibility, it is rare that vsw uses snappy.
|
|
37
37
|
# Uncomment the following if used and python-snappy is installed.
|
|
@@ -159,7 +159,7 @@ class VDataMeta:
|
|
|
159
159
|
>>> meta = VDataReader.getMeta(file)
|
|
160
160
|
>>> info = meta.extendedInfo()
|
|
161
161
|
'''
|
|
162
|
-
def __init__(self, formatVersion=0, compressMethod=0, encryptMethod=0, blocksCount=0, storageStartTime=0, storageEndTime=0, queryStartTime=0, queryEndTime=0, extendedInfo=None):
|
|
162
|
+
def __init__(self, formatVersion=0, compressMethod=0, encryptMethod=0, blocksCount=0, storageStartTime=0, storageEndTime=0, queryStartTime=0, queryEndTime=0, extendedInfo=None, pageSizeCode=0):
|
|
163
163
|
'''
|
|
164
164
|
The constructor for the meta info. For valid values see VData format specification.
|
|
165
165
|
|
|
@@ -183,6 +183,8 @@ class VDataMeta:
|
|
|
183
183
|
self._queryStartTime = queryStartTime
|
|
184
184
|
self._queryEndTime = queryEndTime
|
|
185
185
|
self._extendedInfo = extendedInfo
|
|
186
|
+
self._pageSizeCode = pageSizeCode
|
|
187
|
+
|
|
186
188
|
|
|
187
189
|
def __repr__(self):
|
|
188
190
|
'''
|
|
@@ -276,6 +278,24 @@ class VDataMeta:
|
|
|
276
278
|
'''
|
|
277
279
|
return self._extendedInfo
|
|
278
280
|
|
|
281
|
+
def getPageSize(self):
|
|
282
|
+
'''
|
|
283
|
+
get the page size. default is 64k
|
|
284
|
+
'''
|
|
285
|
+
|
|
286
|
+
if self._pageSizeCode == 0: #default 64k
|
|
287
|
+
return 0x10000
|
|
288
|
+
if self._pageSizeCode == 1: #32k
|
|
289
|
+
return 0x8000
|
|
290
|
+
if self._pageSizeCode == 2: #16k
|
|
291
|
+
return 0x4000
|
|
292
|
+
if self._pageSizeCode == 3: #8k
|
|
293
|
+
return 0x2000
|
|
294
|
+
if self._pageSizeCode == 4: #128k
|
|
295
|
+
return 0x20000
|
|
296
|
+
|
|
297
|
+
return 0x10000
|
|
298
|
+
|
|
279
299
|
class VDataBucket(object):
|
|
280
300
|
def __init__(self):
|
|
281
301
|
self.startTime = 0
|
|
@@ -857,15 +877,22 @@ class VDataFormulaFactory:
|
|
|
857
877
|
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
858
878
|
|
|
859
879
|
class VDataSeriesMeta:
|
|
860
|
-
def __init__(self, formula, storageid, seriesid):
|
|
880
|
+
def __init__(self, formula, storageid, seriesid, pageSize=0x10000):
|
|
861
881
|
self._name = formula._name
|
|
862
882
|
self._formula = formula
|
|
863
883
|
self._seriesid = seriesid
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
884
|
+
if (pageSize != 0x8000):
|
|
885
|
+
self._pageIndex = storageid >> 48 # 48-64-bit
|
|
886
|
+
self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
|
|
887
|
+
self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
|
|
888
|
+
self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
|
|
889
|
+
else:
|
|
890
|
+
self._pageIndex = (storageid >> 35 ) & 0x1F # 35-40 5bits
|
|
891
|
+
self._pageOffset = (storageid >> 20 ) & 0x7FFF # 20-34 15bits
|
|
892
|
+
self._dataLength = ((storageid >> 9) & 0x07FF) #9-19 , 11 bits
|
|
893
|
+
self._itemCount = (storageid & 0x1ff) #0 - 8, 9 bits
|
|
894
|
+
|
|
895
|
+
|
|
869
896
|
def getSeriesId(self):
|
|
870
897
|
return self._seriesid
|
|
871
898
|
|
|
@@ -1016,6 +1043,95 @@ class Float64Tracker:
|
|
|
1016
1043
|
def longBitsToDouble(self, bits):
|
|
1017
1044
|
return _struct.unpack('d', _struct.pack('Q', bits))[0]
|
|
1018
1045
|
|
|
1046
|
+
|
|
1047
|
+
class UInt64Tracker:
|
|
1048
|
+
def __init__(self, formula):
|
|
1049
|
+
self.formula = formula
|
|
1050
|
+
self.bitValue = 0
|
|
1051
|
+
self.signalValue = 0
|
|
1052
|
+
self.leadingZeros = 0
|
|
1053
|
+
self.trailingZeros = 0
|
|
1054
|
+
self.updatedBits = False
|
|
1055
|
+
|
|
1056
|
+
def getBitValue(self):
|
|
1057
|
+
return self.bitValue
|
|
1058
|
+
|
|
1059
|
+
def setBitValue(self, value):
|
|
1060
|
+
self.bitValue = value
|
|
1061
|
+
self.updatedBits = True
|
|
1062
|
+
|
|
1063
|
+
def getSignalValue(self):
|
|
1064
|
+
if (self.updatedBits):
|
|
1065
|
+
self.signalValue = self.longBitsToDouble(self.bitValue)
|
|
1066
|
+
self.updatedBits = False
|
|
1067
|
+
return self.signalValue
|
|
1068
|
+
|
|
1069
|
+
def setSignalValue(self, value):
|
|
1070
|
+
self.signalValue = value
|
|
1071
|
+
self.updatedBits = False
|
|
1072
|
+
|
|
1073
|
+
def getPhysicalValue(self):
|
|
1074
|
+
return self.formula.apply(self.getSignalValue())
|
|
1075
|
+
|
|
1076
|
+
def getLeadingZeros(self):
|
|
1077
|
+
return self.leadingZeros
|
|
1078
|
+
|
|
1079
|
+
def setLeadingZeros(self, value):
|
|
1080
|
+
self.leadingZeros = value
|
|
1081
|
+
|
|
1082
|
+
def getTrailingZeros(self):
|
|
1083
|
+
return self.trailingZeros
|
|
1084
|
+
|
|
1085
|
+
def setTrailingZeros(self, _value):
|
|
1086
|
+
self.trailingZeros = _value
|
|
1087
|
+
|
|
1088
|
+
def longBitsToDouble(self, bits):
|
|
1089
|
+
return bits #_struct.unpack('d', _struct.pack('Q', bits))[0]
|
|
1090
|
+
|
|
1091
|
+
class Int64Tracker:
|
|
1092
|
+
def __init__(self, formula):
|
|
1093
|
+
self.formula = formula
|
|
1094
|
+
self.bitValue = 0
|
|
1095
|
+
self.signalValue = 0
|
|
1096
|
+
self.leadingZeros = 0
|
|
1097
|
+
self.trailingZeros = 0
|
|
1098
|
+
self.updatedBits = False
|
|
1099
|
+
|
|
1100
|
+
def getBitValue(self):
|
|
1101
|
+
return self.bitValue
|
|
1102
|
+
|
|
1103
|
+
def setBitValue(self, value):
|
|
1104
|
+
self.bitValue = value
|
|
1105
|
+
self.updatedBits = True
|
|
1106
|
+
|
|
1107
|
+
def getSignalValue(self):
|
|
1108
|
+
if (self.updatedBits):
|
|
1109
|
+
self.signalValue = self.longBitsToDouble(self.bitValue)
|
|
1110
|
+
self.updatedBits = False
|
|
1111
|
+
return self.signalValue
|
|
1112
|
+
|
|
1113
|
+
def setSignalValue(self, value):
|
|
1114
|
+
self.signalValue = value
|
|
1115
|
+
self.updatedBits = False
|
|
1116
|
+
|
|
1117
|
+
def getPhysicalValue(self):
|
|
1118
|
+
return self.formula.apply(self.getSignalValue())
|
|
1119
|
+
|
|
1120
|
+
def getLeadingZeros(self):
|
|
1121
|
+
return self.leadingZeros
|
|
1122
|
+
|
|
1123
|
+
def setLeadingZeros(self, value):
|
|
1124
|
+
self.leadingZeros = value
|
|
1125
|
+
|
|
1126
|
+
def getTrailingZeros(self):
|
|
1127
|
+
return self.trailingZeros
|
|
1128
|
+
|
|
1129
|
+
def setTrailingZeros(self, _value):
|
|
1130
|
+
self.trailingZeros = _value
|
|
1131
|
+
|
|
1132
|
+
def longBitsToDouble(self, bits):
|
|
1133
|
+
return _struct.unpack('q', _struct.pack('Q', bits))[0]
|
|
1134
|
+
|
|
1019
1135
|
class IntTracker:
|
|
1020
1136
|
def __init__(self, formula):
|
|
1021
1137
|
self.formula = formula
|
|
@@ -1081,6 +1197,7 @@ class DType:
|
|
|
1081
1197
|
Struct = 9
|
|
1082
1198
|
Float32 = 10
|
|
1083
1199
|
Blob = 11
|
|
1200
|
+
UInt64 = 12
|
|
1084
1201
|
|
|
1085
1202
|
class DEncoder:
|
|
1086
1203
|
NONE = 0
|
|
@@ -1397,6 +1514,55 @@ class VDataFloat64ErasureDecode:
|
|
|
1397
1514
|
def bits(self):
|
|
1398
1515
|
return self.fltTracker.getBitValue()
|
|
1399
1516
|
|
|
1517
|
+
class VDataUint64XorDecode:
|
|
1518
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1519
|
+
self.page = _page
|
|
1520
|
+
self.bitTracker = _bitTracker
|
|
1521
|
+
self.intTracker = UInt64Tracker(_formula)
|
|
1522
|
+
|
|
1523
|
+
def first(self):
|
|
1524
|
+
self.next()
|
|
1525
|
+
|
|
1526
|
+
def next(self):
|
|
1527
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
|
|
1528
|
+
return
|
|
1529
|
+
|
|
1530
|
+
xorValue = 0
|
|
1531
|
+
|
|
1532
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0): #use previous block info
|
|
1533
|
+
previousLeadingZeros = self.intTracker.getLeadingZeros()
|
|
1534
|
+
previousTrailingZeros = self.intTracker.getTrailingZeros()
|
|
1535
|
+
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - previousLeadingZeros - previousTrailingZeros, self.bitTracker)
|
|
1536
|
+
xorValue = xorValue << previousTrailingZeros
|
|
1537
|
+
else:
|
|
1538
|
+
leadingZeros = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_LEADING_ZERO_LENGTH_BITS, self.bitTracker)
|
|
1539
|
+
blockSize = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_BLOCK_SIZE_LENGTH_BITS, self.bitTracker) + VDataConstants.XOR_BLOCK_SIZE_ADJUSTMENTS
|
|
1540
|
+
|
|
1541
|
+
trailingZeros = 64 - blockSize - leadingZeros
|
|
1542
|
+
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, blockSize, self.bitTracker)
|
|
1543
|
+
xorValue = xorValue << trailingZeros
|
|
1544
|
+
|
|
1545
|
+
self.intTracker.setLeadingZeros(leadingZeros)
|
|
1546
|
+
self.intTracker.setTrailingZeros(trailingZeros)
|
|
1547
|
+
|
|
1548
|
+
value = xorValue ^ self.intTracker.getBitValue()
|
|
1549
|
+
self.intTracker.setBitValue(value)
|
|
1550
|
+
|
|
1551
|
+
def get(self):
|
|
1552
|
+
return self.intTracker.getPhysicalValue()
|
|
1553
|
+
|
|
1554
|
+
def raw(self):
|
|
1555
|
+
return self.intTracker.getSignalValue()
|
|
1556
|
+
|
|
1557
|
+
def bits(self):
|
|
1558
|
+
return self.intTracker.getBitValue()
|
|
1559
|
+
|
|
1560
|
+
class VDataInt64XorDecode(VDataUint64XorDecode):
|
|
1561
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1562
|
+
self.page = _page
|
|
1563
|
+
self.bitTracker = _bitTracker
|
|
1564
|
+
self.intTracker = Int64Tracker(_formula)
|
|
1565
|
+
|
|
1400
1566
|
class VDataFloat64XorDecode:
|
|
1401
1567
|
def __init__(self, _page, _formula, _bitTracker):
|
|
1402
1568
|
self.page = _page
|
|
@@ -1666,7 +1832,7 @@ class VDataNumberArrayDecode:
|
|
|
1666
1832
|
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, fn, _bitTracker))
|
|
1667
1833
|
else:
|
|
1668
1834
|
if (_length != flen and _matchLength):
|
|
1669
|
-
raise Exception('FORMAT_VDATA_ARRAY_LENGTH_MISMATCH: length=' + str(_length) + ', formula is ' + _formula.
|
|
1835
|
+
raise Exception('FORMAT_VDATA_ARRAY_LENGTH_MISMATCH: length=' + str(_length) + ', formula is ' + _formula._name)
|
|
1670
1836
|
s = _length if _length < flen else flen
|
|
1671
1837
|
for i in range(s):
|
|
1672
1838
|
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[i], _bitTracker))
|
|
@@ -1682,7 +1848,7 @@ class VDataNumberArrayDecode:
|
|
|
1682
1848
|
return VDataUInt8XorDecode(_page, _formula, _bitTracker)
|
|
1683
1849
|
else:
|
|
1684
1850
|
return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
|
|
1685
|
-
if (_dtype == DType.Float64
|
|
1851
|
+
if (_dtype == DType.Float64 ):
|
|
1686
1852
|
if (_encoder == DEncoder.XOR):
|
|
1687
1853
|
return VDataFloat64XorDecode(_page, _formula, _bitTracker)
|
|
1688
1854
|
elif (_encoder == DEncoder.CHIMP):
|
|
@@ -1701,6 +1867,10 @@ class VDataNumberArrayDecode:
|
|
|
1701
1867
|
return VDataBooleanNoneDecode(_page, _formula, _bitTracker)
|
|
1702
1868
|
if (_dtype == DType.Float32):
|
|
1703
1869
|
return VDataFloat32NoneDecode(_page, _formula, _bitTracker)
|
|
1870
|
+
if (_dtype == DType.UInt64):
|
|
1871
|
+
return VDataUint64XorDecode(_page, _formula, _bitTracker)
|
|
1872
|
+
if (_dtype == DType.Int64):
|
|
1873
|
+
return VDataInt64XorDecode(_page, _formula, _bitTracker)
|
|
1704
1874
|
raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(_dtype))
|
|
1705
1875
|
|
|
1706
1876
|
def first(self):
|
|
@@ -1819,7 +1989,7 @@ class VDataStructDecode:
|
|
|
1819
1989
|
return VDataUInt8XorDecode(_page, _formula, _bitTracker)
|
|
1820
1990
|
else:
|
|
1821
1991
|
return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
|
|
1822
|
-
if (_dtype == DType.Float64
|
|
1992
|
+
if (_dtype == DType.Float64 ):
|
|
1823
1993
|
if (_encoder == DEncoder.XOR):
|
|
1824
1994
|
return VDataFloat64XorDecode(_page, _formula, _bitTracker)
|
|
1825
1995
|
elif (_encoder == DEncoder.CHIMP):
|
|
@@ -2134,6 +2304,26 @@ class VDataSeriesDecodeBuilder(object):
|
|
|
2134
2304
|
_subtype = (sub >> 4) & 0x0F
|
|
2135
2305
|
_encoder = sub & 0x0F
|
|
2136
2306
|
_offset = 32
|
|
2307
|
+
|
|
2308
|
+
element_exist_array = []
|
|
2309
|
+
|
|
2310
|
+
if (_version > 27 ):
|
|
2311
|
+
#for nullable element array, load null element map.
|
|
2312
|
+
_byte_length = 2 if _length<=16 else (_length+7)//8
|
|
2313
|
+
_offset = 32 + _byte_length *8
|
|
2314
|
+
_bitmap = btreader.read(_offset)
|
|
2315
|
+
for i in range(_length):
|
|
2316
|
+
element_exist_array.append(True)
|
|
2317
|
+
byte_index = i // 8
|
|
2318
|
+
bit_index = 7- i % 8
|
|
2319
|
+
|
|
2320
|
+
element_exist_array.append( (((_bitmap[byte_index]>> bit_index) & 0x01) == 1 ))
|
|
2321
|
+
|
|
2322
|
+
else :
|
|
2323
|
+
for i in range(_length):
|
|
2324
|
+
element_exist_array.append( True)
|
|
2325
|
+
_offset = 32
|
|
2326
|
+
|
|
2137
2327
|
|
|
2138
2328
|
if (isinstance(_data.getFormula(), VDataFormulaNamedArray)):
|
|
2139
2329
|
return VDataSeriesDecodeNamedArray(_data, _offset, _subtype, _encoder, _length)
|
|
@@ -2319,7 +2509,7 @@ class VDataSeriesDecodeNumber(VDataSeriesDecode):
|
|
|
2319
2509
|
self.valueDecode = VDataUInt8XorDecode(self.page, self.formula, self.bitTracker)
|
|
2320
2510
|
else:
|
|
2321
2511
|
self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
|
|
2322
|
-
elif (dtype == DType.Float64
|
|
2512
|
+
elif (dtype == DType.Float64 ):
|
|
2323
2513
|
if (encoder == DEncoder.XOR):
|
|
2324
2514
|
self.valueDecode = VDataFloat64XorDecode(self.page, self.formula, self.bitTracker)
|
|
2325
2515
|
elif (encoder == DEncoder.CHIMP):
|
|
@@ -2338,6 +2528,10 @@ class VDataSeriesDecodeNumber(VDataSeriesDecode):
|
|
|
2338
2528
|
self.valueDecode = VDataBooleanNoneDecode(self.page, self.formula, self.bitTracker)
|
|
2339
2529
|
elif (dtype == DType.Float32):
|
|
2340
2530
|
self.valueDecode = VDataFloat32NoneDecode(self.page, self.formula, self.bitTracker)
|
|
2531
|
+
elif (dtype == DType.UInt64):
|
|
2532
|
+
self.valueDecode = VDataUint64XorDecode(self.page, self.formula, self.bitTracker)
|
|
2533
|
+
elif (dtype == DType.Int64):
|
|
2534
|
+
self.valueDecode = VDataInt64XorDecode(self.page, self.formula, self.bitTracker)
|
|
2341
2535
|
else:
|
|
2342
2536
|
raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(dtype))
|
|
2343
2537
|
|
|
@@ -2431,7 +2625,7 @@ class VDataSeriesDecodeNamedArray(VDataSeriesDecode):
|
|
|
2431
2625
|
|
|
2432
2626
|
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2433
2627
|
if (dtype != DType.String):
|
|
2434
|
-
self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length,
|
|
2628
|
+
self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, False, self.formula, self.bitTracker)
|
|
2435
2629
|
else:
|
|
2436
2630
|
self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, True, self.formula, self.bitTracker)
|
|
2437
2631
|
|
|
@@ -3342,6 +3536,8 @@ class VDataFrame:
|
|
|
3342
3536
|
Returns:
|
|
3343
3537
|
array_like(array_like)
|
|
3344
3538
|
'''
|
|
3539
|
+
if self._endTime - self._startTime > 3600000:
|
|
3540
|
+
return self.values(densifyRowsAhead, densifyOutputItv)
|
|
3345
3541
|
oread = None
|
|
3346
3542
|
if "all" == self._signalQueueMode:
|
|
3347
3543
|
oread = VDataObjectsSeriesAllMode(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
|
|
@@ -3502,6 +3698,11 @@ class VDataDecoder:
|
|
|
3502
3698
|
self._columnExpandMode = VDataUtils.getColumnExpandMode(self._columnExpandMode)
|
|
3503
3699
|
self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(self._columnExpandMode)
|
|
3504
3700
|
|
|
3701
|
+
if self._queryStartTime ==0 and self._meta.queryStartTime() ==0 :
|
|
3702
|
+
self._queryStartTime = self._meta.storageStartTime()
|
|
3703
|
+
if self._queryEndTime ==0 and self._meta.queryEndTime() ==0 :
|
|
3704
|
+
self._queryEndTime = self._meta.storageEndTime()
|
|
3705
|
+
|
|
3505
3706
|
formatVersion = self._meta.formatVersion()
|
|
3506
3707
|
compressMethod = self._meta.compressMethod()
|
|
3507
3708
|
# encryptMethod = self._meta.encryptMethod()
|
|
@@ -3513,7 +3714,10 @@ class VDataDecoder:
|
|
|
3513
3714
|
logger.debug("file query start time: " + str(self._meta.queryStartTime()))
|
|
3514
3715
|
logger.debug("file query end time: " + str(self._meta.queryEndTime()))
|
|
3515
3716
|
|
|
3516
|
-
self.
|
|
3717
|
+
if self._meta.getPageSize() != 0x8000:
|
|
3718
|
+
self.readKeys()
|
|
3719
|
+
else:
|
|
3720
|
+
self.readStfKeys()
|
|
3517
3721
|
|
|
3518
3722
|
for name, key in self._keys.items():
|
|
3519
3723
|
formulas[key._seriesid] = key._formula
|
|
@@ -3558,7 +3762,8 @@ class VDataDecoder:
|
|
|
3558
3762
|
bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3559
3763
|
length = _struct.unpack('<i', self._reader.read(4))[0] - 40 #include header
|
|
3560
3764
|
self._reader.read(4) #ignore
|
|
3561
|
-
|
|
3765
|
+
elif (self._meta.getPageSize () != 0x8000):
|
|
3766
|
+
#default 64 page
|
|
3562
3767
|
crc = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3563
3768
|
living = self._reader.read(1)[0] & 1
|
|
3564
3769
|
self._reader.read(3) #ignore
|
|
@@ -3573,6 +3778,19 @@ class VDataDecoder:
|
|
|
3573
3778
|
blobCrc = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3574
3779
|
blobCount = _struct.unpack('<i', self._reader.read(4))[0]
|
|
3575
3780
|
blobDataLength = _struct.unpack('<i', self._reader.read(4))[0]
|
|
3781
|
+
else:
|
|
3782
|
+
# 32k page
|
|
3783
|
+
crc = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3784
|
+
living = self._reader.read(1)[0] & 1
|
|
3785
|
+
self._reader.read(3) #ignore
|
|
3786
|
+
bucketCycle = _struct.unpack('<i', self._reader.read(4))[0]
|
|
3787
|
+
bucketStartTime = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3788
|
+
bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3789
|
+
length = _struct.unpack('<i', self._reader.read(4))[0] - 48 #include header
|
|
3790
|
+
|
|
3791
|
+
position = _struct.unpack('<i', self._reader.read(4))[0]
|
|
3792
|
+
blobCount = _struct.unpack('<i', self._reader.read(4))[0]
|
|
3793
|
+
blobDataLength = _struct.unpack('<i', self._reader.read(4))[0]
|
|
3576
3794
|
|
|
3577
3795
|
if (length <= 0):
|
|
3578
3796
|
raise Exception('FORMAT_VDATA_BUCKET_LENGTH_INVALID')
|
|
@@ -3599,12 +3817,26 @@ class VDataDecoder:
|
|
|
3599
3817
|
if (activePages <= 0):
|
|
3600
3818
|
raise Exception('FORMAT_VDATA_ACTIVE_PAGES_ZERO')
|
|
3601
3819
|
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3820
|
+
if (self._meta.getPageSize() != 0x8000):
|
|
3821
|
+
for i in range(seriesCount):
|
|
3822
|
+
seriesIds.append(_struct.unpack('<i', databytes[index:index+4])[0])
|
|
3823
|
+
index += 4
|
|
3824
|
+
for i in range(seriesCount):
|
|
3825
|
+
storageIds.append(_struct.unpack('<q', databytes[index:index+8])[0])
|
|
3826
|
+
index += 8
|
|
3827
|
+
else:
|
|
3828
|
+
#5 byte storageId for 32k page size vsw
|
|
3829
|
+
for i in range(len(self._keyIds)):
|
|
3830
|
+
if (((databytes [index+(i>>3)] >> (i&0x7)) & 0x01) == 1):
|
|
3831
|
+
seriesIds.append(self._keyIds[i])
|
|
3832
|
+
|
|
3833
|
+
index = (len(self._keyIds) +7 )//8 + index
|
|
3834
|
+
for i in range(seriesCount):
|
|
3835
|
+
storageId = _struct.unpack('<i', databytes[index:index+4])[0]
|
|
3836
|
+
storageId = storageId& 0xffffffff
|
|
3837
|
+
storageId = ((databytes[index+4] &0xff) << 32 ) + storageId
|
|
3838
|
+
storageIds.append(storageId)
|
|
3839
|
+
index += 5
|
|
3608
3840
|
|
|
3609
3841
|
blobSeriesIds =[]
|
|
3610
3842
|
blobOffsetVec =[]
|
|
@@ -3626,7 +3858,7 @@ class VDataDecoder:
|
|
|
3626
3858
|
raise Exception('FORMAT_VDATA_SERIES_NOT_FOUND' + str(seriesIds[i]))
|
|
3627
3859
|
if (hasSignalFilter and not VDataSeriesUtils.serieMatchSignalFilter(self._signals, self._prefixes, formulas[seriesIds[i]], self._columnExpandMode, self._qualifiedExpandName)):
|
|
3628
3860
|
continue
|
|
3629
|
-
mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i )
|
|
3861
|
+
mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i , self._meta.getPageSize())
|
|
3630
3862
|
if (mt._dataLength == 0 or mt._itemCount == 0):
|
|
3631
3863
|
raise Exception('FORMAT_VDATA_SERIES_DATA_EMPTY')
|
|
3632
3864
|
|
|
@@ -3646,7 +3878,7 @@ class VDataDecoder:
|
|
|
3646
3878
|
|
|
3647
3879
|
for i in range(len(seriesMetas)):
|
|
3648
3880
|
mt = seriesMetas[i]
|
|
3649
|
-
bstart = index + mt._pageIndex *
|
|
3881
|
+
bstart = index + mt._pageIndex * self._meta.getPageSize() + mt._pageOffset
|
|
3650
3882
|
bend = bstart + mt._dataLength
|
|
3651
3883
|
bt = databytes[bstart:bend]
|
|
3652
3884
|
|
|
@@ -3706,6 +3938,24 @@ class VDataDecoder:
|
|
|
3706
3938
|
logger.warn("Replace series " + str(keyid) + " duplicate name {" + formula._name + "} with name {" + sname + "}")
|
|
3707
3939
|
formula._name = sname
|
|
3708
3940
|
self._keys[sname] = VDataSeriesSpec(sname, keyid, formula, keydesc)
|
|
3941
|
+
|
|
3942
|
+
def readStfKeys(self):
|
|
3943
|
+
(keysCount, length) = _struct.unpack('<ii', self._reader.read(8))
|
|
3944
|
+
if (keysCount <= 0):
|
|
3945
|
+
raise Exception('FORMAT_VDATA_KEYS_COUNT_INVALID')
|
|
3946
|
+
if (length <= 0):
|
|
3947
|
+
raise Exception('FORMAT_VDATA_KEYS_LENGTH_INVALID')
|
|
3948
|
+
databytes = self._reader.read(length)
|
|
3949
|
+
noneFormula = VDataFormulaNone('keys')
|
|
3950
|
+
decode = VDataInt32DeltaDecode(databytes, noneFormula, BitTracker())
|
|
3951
|
+
|
|
3952
|
+
self._keyIds = {}
|
|
3953
|
+
for i in range (keysCount):
|
|
3954
|
+
decode.next()
|
|
3955
|
+
self._keyIds[i] = decode.get()
|
|
3956
|
+
sname ="0x%08X"%self._keyIds[i]
|
|
3957
|
+
self._keys[sname] = VDataSeriesSpec(sname, self._keyIds[i] , VDataFormulaNone(sname), '')
|
|
3958
|
+
|
|
3709
3959
|
|
|
3710
3960
|
def getUniqueName(self, keys, key):
|
|
3711
3961
|
if key not in keys:
|
|
@@ -4094,7 +4344,7 @@ class VDataObjectsSeriesReader:
|
|
|
4094
4344
|
time = decoder.time()
|
|
4095
4345
|
millis = decoder.timestamp()
|
|
4096
4346
|
pval = decoder.value()
|
|
4097
|
-
if (millis >=
|
|
4347
|
+
if (millis >= decoder.qstime and millis <= decoder.qetime):
|
|
4098
4348
|
rowidx = millis - self._startTime
|
|
4099
4349
|
if (rawobjs[rowidx] is None):
|
|
4100
4350
|
list = []
|
|
@@ -4137,7 +4387,7 @@ class VDataObjectsSeriesReader:
|
|
|
4137
4387
|
time = decoder.time()
|
|
4138
4388
|
millis = decoder.timestamp()
|
|
4139
4389
|
pval = decoder.value()
|
|
4140
|
-
if (millis >=
|
|
4390
|
+
if (millis >= decoder.qstime and millis <= decoder.qetime):
|
|
4141
4391
|
rowidx = millis - self._startTime
|
|
4142
4392
|
if (rawobjs[rowidx] is None):
|
|
4143
4393
|
list = []
|
|
@@ -4198,12 +4448,11 @@ class VDataObjectsSeriesReader:
|
|
|
4198
4448
|
rawobjs = [None] * rows # 2-d array of millisecond + data
|
|
4199
4449
|
for i in range(rows):
|
|
4200
4450
|
rawobjs[i] = [None] * self._colsize
|
|
4201
|
-
rawobjs[i][0] = (startFreqBase + i) * frequency
|
|
4202
|
-
|
|
4203
|
-
rowidx = 0
|
|
4204
|
-
lastrowidx = -1
|
|
4451
|
+
rawobjs[i][0] = (startFreqBase + i) * frequency
|
|
4205
4452
|
|
|
4206
4453
|
for pname in self._series:
|
|
4454
|
+
rowidx = 0
|
|
4455
|
+
lastrowidx = -1
|
|
4207
4456
|
serie = self._series[pname]
|
|
4208
4457
|
colidx = self._indices.get(serie.name())
|
|
4209
4458
|
decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
|
|
@@ -4243,10 +4492,10 @@ class VDataObjectsSeriesReader:
|
|
|
4243
4492
|
rawobjs[i] = [None] * self._colsize
|
|
4244
4493
|
rawobjs[i][0] = (startFreqBase + i) * frequency
|
|
4245
4494
|
|
|
4246
|
-
rowidx = 0
|
|
4247
|
-
lastrowidx = -1;
|
|
4495
|
+
rowidx = 0
|
|
4248
4496
|
|
|
4249
4497
|
for pname in self._series:
|
|
4498
|
+
lastrowidx = -1
|
|
4250
4499
|
serie = self._series[pname]
|
|
4251
4500
|
colidx = self._indices.get(serie.name())
|
|
4252
4501
|
decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
|
|
@@ -4764,6 +5013,7 @@ class VDataReader:
|
|
|
4764
5013
|
qstime = 0 # query start time
|
|
4765
5014
|
qetime = 0 # query end time
|
|
4766
5015
|
extlength = 0
|
|
5016
|
+
pageSizeCode = 0
|
|
4767
5017
|
|
|
4768
5018
|
if (formatVersion > 0):
|
|
4769
5019
|
if (formatVersion < 26):
|
|
@@ -4771,10 +5021,14 @@ class VDataReader:
|
|
|
4771
5021
|
else:
|
|
4772
5022
|
(sstime, setime, qstime, qetime) = _struct.unpack('<qqqq', _reader.read(32))
|
|
4773
5023
|
extlength = _struct.unpack('<H', _reader.read(2))[0]
|
|
4774
|
-
|
|
5024
|
+
if (formatVersion > 26):
|
|
5025
|
+
pageSizeCode = _reader.read(1)[0]
|
|
5026
|
+
_reader.read(5) # reserved
|
|
5027
|
+
else:
|
|
5028
|
+
_reader.read(6) # reserved
|
|
4775
5029
|
|
|
4776
5030
|
extbytes = _reader.read(extlength) if (extlength > 0) else []
|
|
4777
|
-
return VDataMeta(formatVersion=formatVersion, compressMethod=compressMethod, encryptMethod=encryptMethod, blocksCount=blocksCount, storageStartTime=sstime, storageEndTime=setime, queryStartTime=qstime, queryEndTime=qetime, extendedInfo=extbytes)
|
|
5031
|
+
return VDataMeta(formatVersion=formatVersion, compressMethod=compressMethod, encryptMethod=encryptMethod, blocksCount=blocksCount, storageStartTime=sstime, storageEndTime=setime, queryStartTime=qstime, queryEndTime=qetime, extendedInfo=extbytes, pageSizeCode = pageSizeCode)
|
|
4778
5032
|
|
|
4779
5033
|
|
|
4780
5034
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|