exceeddata-sdk-vdata 2.9.3.2__tar.gz → 2.9.6.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,29 @@
1
+ Copyright (C) 2016-2024 Smart Software for Car Technologies Inc. and EXCEEDDATA
2
+ https://www.smartsct.com
3
+ https://www.exceeddata.com
4
+
5
+ MIT License
6
+
7
+ Permission is hereby granted, free of charge, to any person obtaining
8
+ a copy of this software and associated documentation files (the
9
+ "Software"), to deal in the Software without restriction, including
10
+ without limitation the rights to use, copy, modify, merge, publish,
11
+ distribute, sublicense, and/or sell copies of the Software, and to
12
+ permit persons to whom the Software is furnished to do so, subject to
13
+ the following conditions:
14
+
15
+ The above copyright notice and this permission notice shall be
16
+ included in all copies or substantial portions of the Software.
17
+
18
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
22
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
+
26
+ Except as contained in this notice, the name of a copyright holder
27
+ shall not be used in advertising or otherwise to promote the sale, use
28
+ or other dealings in this Software without prior written authorization
29
+ of the copyright holder.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.9.3.2
3
+ Version: 2.9.6.1
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -32,6 +32,6 @@
32
32
 
33
33
  name = 'vdata'
34
34
 
35
- __version__ = "2.9.3.2"
35
+ __version__ = "2.9.6.1"
36
36
  __author__ = 'Nick Xie'
37
37
  __credits__ = 'Smart Software for Car Technologies Inc'
@@ -31,7 +31,7 @@
31
31
  # of the copyright holder.
32
32
  #
33
33
  # Author: Nick Xie
34
- # Version: 2.9.3.2
34
+ # Version: 2.9.6.1
35
35
  #
36
36
  # Note: for specification flag compatibility, it is rare that vsw uses snappy.
37
37
  # Uncomment the following if used and python-snappy is installed.
@@ -159,7 +159,7 @@ class VDataMeta:
159
159
  >>> meta = VDataReader.getMeta(file)
160
160
  >>> info = meta.extendedInfo()
161
161
  '''
162
- def __init__(self, formatVersion=0, compressMethod=0, encryptMethod=0, blocksCount=0, storageStartTime=0, storageEndTime=0, queryStartTime=0, queryEndTime=0, extendedInfo=None):
162
+ def __init__(self, formatVersion=0, compressMethod=0, encryptMethod=0, blocksCount=0, storageStartTime=0, storageEndTime=0, queryStartTime=0, queryEndTime=0, extendedInfo=None, pageSizeCode=0):
163
163
  '''
164
164
  The constructor for the meta info. For valid values see VData format specification.
165
165
 
@@ -183,6 +183,8 @@ class VDataMeta:
183
183
  self._queryStartTime = queryStartTime
184
184
  self._queryEndTime = queryEndTime
185
185
  self._extendedInfo = extendedInfo
186
+ self._pageSizeCode = pageSizeCode
187
+
186
188
 
187
189
  def __repr__(self):
188
190
  '''
@@ -276,6 +278,24 @@ class VDataMeta:
276
278
  '''
277
279
  return self._extendedInfo
278
280
 
281
+ def getPageSize(self):
282
+ '''
283
+ get the page size. default is 64k
284
+ '''
285
+
286
+ if self._pageSizeCode == 0: #default 64k
287
+ return 0x10000
288
+ if self._pageSizeCode == 1: #32k
289
+ return 0x8000
290
+ if self._pageSizeCode == 2: #16k
291
+ return 0x4000
292
+ if self._pageSizeCode == 3: #8k
293
+ return 0x2000
294
+ if self._pageSizeCode == 4: #128k
295
+ return 0x20000
296
+
297
+ return 0x10000
298
+
279
299
  class VDataBucket(object):
280
300
  def __init__(self):
281
301
  self.startTime = 0
@@ -857,15 +877,22 @@ class VDataFormulaFactory:
857
877
  raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
858
878
 
859
879
  class VDataSeriesMeta:
860
- def __init__(self, formula, storageid, seriesid):
880
+ def __init__(self, formula, storageid, seriesid, pageSize=0x10000):
861
881
  self._name = formula._name
862
882
  self._formula = formula
863
883
  self._seriesid = seriesid
864
- self._pageIndex = storageid >> 48 # 48-64-bit
865
- self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
866
- self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
867
- self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
868
-
884
+ if (pageSize != 0x8000):
885
+ self._pageIndex = storageid >> 48 # 48-64-bit
886
+ self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
887
+ self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
888
+ self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
889
+ else:
890
+ self._pageIndex = (storageid >> 35 ) & 0x1F # 35-40 5bits
891
+ self._pageOffset = (storageid >> 20 ) & 0x7FFF # 20-34 15bits
892
+ self._dataLength = ((storageid >> 9) & 0x07FF) #9-19 , 11 bits
893
+ self._itemCount = (storageid & 0x1ff) #0 - 8, 9 bits
894
+
895
+
869
896
  def getSeriesId(self):
870
897
  return self._seriesid
871
898
 
@@ -1016,6 +1043,95 @@ class Float64Tracker:
1016
1043
  def longBitsToDouble(self, bits):
1017
1044
  return _struct.unpack('d', _struct.pack('Q', bits))[0]
1018
1045
 
1046
+
1047
+ class UInt64Tracker:
1048
+ def __init__(self, formula):
1049
+ self.formula = formula
1050
+ self.bitValue = 0
1051
+ self.signalValue = 0
1052
+ self.leadingZeros = 0
1053
+ self.trailingZeros = 0
1054
+ self.updatedBits = False
1055
+
1056
+ def getBitValue(self):
1057
+ return self.bitValue
1058
+
1059
+ def setBitValue(self, value):
1060
+ self.bitValue = value
1061
+ self.updatedBits = True
1062
+
1063
+ def getSignalValue(self):
1064
+ if (self.updatedBits):
1065
+ self.signalValue = self.longBitsToDouble(self.bitValue)
1066
+ self.updatedBits = False
1067
+ return self.signalValue
1068
+
1069
+ def setSignalValue(self, value):
1070
+ self.signalValue = value
1071
+ self.updatedBits = False
1072
+
1073
+ def getPhysicalValue(self):
1074
+ return self.formula.apply(self.getSignalValue())
1075
+
1076
+ def getLeadingZeros(self):
1077
+ return self.leadingZeros
1078
+
1079
+ def setLeadingZeros(self, value):
1080
+ self.leadingZeros = value
1081
+
1082
+ def getTrailingZeros(self):
1083
+ return self.trailingZeros
1084
+
1085
+ def setTrailingZeros(self, _value):
1086
+ self.trailingZeros = _value
1087
+
1088
+ def longBitsToDouble(self, bits):
1089
+ return bits #_struct.unpack('d', _struct.pack('Q', bits))[0]
1090
+
1091
+ class Int64Tracker:
1092
+ def __init__(self, formula):
1093
+ self.formula = formula
1094
+ self.bitValue = 0
1095
+ self.signalValue = 0
1096
+ self.leadingZeros = 0
1097
+ self.trailingZeros = 0
1098
+ self.updatedBits = False
1099
+
1100
+ def getBitValue(self):
1101
+ return self.bitValue
1102
+
1103
+ def setBitValue(self, value):
1104
+ self.bitValue = value
1105
+ self.updatedBits = True
1106
+
1107
+ def getSignalValue(self):
1108
+ if (self.updatedBits):
1109
+ self.signalValue = self.longBitsToDouble(self.bitValue)
1110
+ self.updatedBits = False
1111
+ return self.signalValue
1112
+
1113
+ def setSignalValue(self, value):
1114
+ self.signalValue = value
1115
+ self.updatedBits = False
1116
+
1117
+ def getPhysicalValue(self):
1118
+ return self.formula.apply(self.getSignalValue())
1119
+
1120
+ def getLeadingZeros(self):
1121
+ return self.leadingZeros
1122
+
1123
+ def setLeadingZeros(self, value):
1124
+ self.leadingZeros = value
1125
+
1126
+ def getTrailingZeros(self):
1127
+ return self.trailingZeros
1128
+
1129
+ def setTrailingZeros(self, _value):
1130
+ self.trailingZeros = _value
1131
+
1132
+ def longBitsToDouble(self, bits):
1133
+ return _struct.unpack('q', _struct.pack('Q', bits))[0]
1134
+
1019
1135
  class IntTracker:
1020
1136
  def __init__(self, formula):
1021
1137
  self.formula = formula
@@ -1081,6 +1197,7 @@ class DType:
1081
1197
  Struct = 9
1082
1198
  Float32 = 10
1083
1199
  Blob = 11
1200
+ UInt64 = 12
1084
1201
 
1085
1202
  class DEncoder:
1086
1203
  NONE = 0
@@ -1397,6 +1514,55 @@ class VDataFloat64ErasureDecode:
1397
1514
  def bits(self):
1398
1515
  return self.fltTracker.getBitValue()
1399
1516
 
1517
+ class VDataUint64XorDecode:
1518
+ def __init__(self, _page, _formula, _bitTracker):
1519
+ self.page = _page
1520
+ self.bitTracker = _bitTracker
1521
+ self.intTracker = UInt64Tracker(_formula)
1522
+
1523
+ def first(self):
1524
+ self.next()
1525
+
1526
+ def next(self):
1527
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1528
+ return
1529
+
1530
+ xorValue = 0
1531
+
1532
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0): #use previous block info
1533
+ previousLeadingZeros = self.intTracker.getLeadingZeros()
1534
+ previousTrailingZeros = self.intTracker.getTrailingZeros()
1535
+ xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - previousLeadingZeros - previousTrailingZeros, self.bitTracker)
1536
+ xorValue = xorValue << previousTrailingZeros
1537
+ else:
1538
+ leadingZeros = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_LEADING_ZERO_LENGTH_BITS, self.bitTracker)
1539
+ blockSize = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_BLOCK_SIZE_LENGTH_BITS, self.bitTracker) + VDataConstants.XOR_BLOCK_SIZE_ADJUSTMENTS
1540
+
1541
+ trailingZeros = 64 - blockSize - leadingZeros
1542
+ xorValue = VDataDecodeUtils.readValueFromBitString(self.page, blockSize, self.bitTracker)
1543
+ xorValue = xorValue << trailingZeros
1544
+
1545
+ self.intTracker.setLeadingZeros(leadingZeros)
1546
+ self.intTracker.setTrailingZeros(trailingZeros)
1547
+
1548
+ value = xorValue ^ self.intTracker.getBitValue()
1549
+ self.intTracker.setBitValue(value)
1550
+
1551
+ def get(self):
1552
+ return self.intTracker.getPhysicalValue()
1553
+
1554
+ def raw(self):
1555
+ return self.intTracker.getSignalValue()
1556
+
1557
+ def bits(self):
1558
+ return self.intTracker.getBitValue()
1559
+
1560
+ class VDataInt64XorDecode(VDataUint64XorDecode):
1561
+ def __init__(self, _page, _formula, _bitTracker):
1562
+ self.page = _page
1563
+ self.bitTracker = _bitTracker
1564
+ self.intTracker = Int64Tracker(_formula)
1565
+
1400
1566
  class VDataFloat64XorDecode:
1401
1567
  def __init__(self, _page, _formula, _bitTracker):
1402
1568
  self.page = _page
@@ -1666,7 +1832,7 @@ class VDataNumberArrayDecode:
1666
1832
  self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, fn, _bitTracker))
1667
1833
  else:
1668
1834
  if (_length != flen and _matchLength):
1669
- raise Exception('FORMAT_VDATA_ARRAY_LENGTH_MISMATCH: length=' + str(_length) + ', formula is ' + _formula.name())
1835
+ raise Exception('FORMAT_VDATA_ARRAY_LENGTH_MISMATCH: length=' + str(_length) + ', formula is ' + _formula._name)
1670
1836
  s = _length if _length < flen else flen
1671
1837
  for i in range(s):
1672
1838
  self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[i], _bitTracker))
@@ -1682,7 +1848,7 @@ class VDataNumberArrayDecode:
1682
1848
  return VDataUInt8XorDecode(_page, _formula, _bitTracker)
1683
1849
  else:
1684
1850
  return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
1685
- if (_dtype == DType.Float64 or _dtype == DType.Int64):
1851
+ if (_dtype == DType.Float64 ):
1686
1852
  if (_encoder == DEncoder.XOR):
1687
1853
  return VDataFloat64XorDecode(_page, _formula, _bitTracker)
1688
1854
  elif (_encoder == DEncoder.CHIMP):
@@ -1701,6 +1867,10 @@ class VDataNumberArrayDecode:
1701
1867
  return VDataBooleanNoneDecode(_page, _formula, _bitTracker)
1702
1868
  if (_dtype == DType.Float32):
1703
1869
  return VDataFloat32NoneDecode(_page, _formula, _bitTracker)
1870
+ if (_dtype == DType.UInt64):
1871
+ return VDataUint64XorDecode(_page, _formula, _bitTracker)
1872
+ if (_dtype == DType.Int64):
1873
+ return VDataInt64XorDecode(_page, _formula, _bitTracker)
1704
1874
  raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(_dtype))
1705
1875
 
1706
1876
  def first(self):
@@ -1819,7 +1989,7 @@ class VDataStructDecode:
1819
1989
  return VDataUInt8XorDecode(_page, _formula, _bitTracker)
1820
1990
  else:
1821
1991
  return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
1822
- if (_dtype == DType.Float64 or _dtype == DType.Int64):
1992
+ if (_dtype == DType.Float64 ):
1823
1993
  if (_encoder == DEncoder.XOR):
1824
1994
  return VDataFloat64XorDecode(_page, _formula, _bitTracker)
1825
1995
  elif (_encoder == DEncoder.CHIMP):
@@ -2134,6 +2304,26 @@ class VDataSeriesDecodeBuilder(object):
2134
2304
  _subtype = (sub >> 4) & 0x0F
2135
2305
  _encoder = sub & 0x0F
2136
2306
  _offset = 32
2307
+
2308
+ element_exist_array = []
2309
+
2310
+ if (_version > 27 ):
2311
+ #for nullable element array, load null element map.
2312
+ _byte_length = 2 if _length<=16 else (_length+7)//8
2313
+ _offset = 32 + _byte_length *8
2314
+ _bitmap = btreader.read(_offset)
2315
+ for i in range(_length):
2316
+ element_exist_array.append(True)
2317
+ byte_index = i // 8
2318
+ bit_index = 7- i % 8
2319
+
2320
+ element_exist_array.append( (((_bitmap[byte_index]>> bit_index) & 0x01) == 1 ))
2321
+
2322
+ else :
2323
+ for i in range(_length):
2324
+ element_exist_array.append( True)
2325
+ _offset = 32
2326
+
2137
2327
 
2138
2328
  if (isinstance(_data.getFormula(), VDataFormulaNamedArray)):
2139
2329
  return VDataSeriesDecodeNamedArray(_data, _offset, _subtype, _encoder, _length)
@@ -2319,7 +2509,7 @@ class VDataSeriesDecodeNumber(VDataSeriesDecode):
2319
2509
  self.valueDecode = VDataUInt8XorDecode(self.page, self.formula, self.bitTracker)
2320
2510
  else:
2321
2511
  self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
2322
- elif (dtype == DType.Float64 or dtype == DType.Int64):
2512
+ elif (dtype == DType.Float64 ):
2323
2513
  if (encoder == DEncoder.XOR):
2324
2514
  self.valueDecode = VDataFloat64XorDecode(self.page, self.formula, self.bitTracker)
2325
2515
  elif (encoder == DEncoder.CHIMP):
@@ -2338,6 +2528,10 @@ class VDataSeriesDecodeNumber(VDataSeriesDecode):
2338
2528
  self.valueDecode = VDataBooleanNoneDecode(self.page, self.formula, self.bitTracker)
2339
2529
  elif (dtype == DType.Float32):
2340
2530
  self.valueDecode = VDataFloat32NoneDecode(self.page, self.formula, self.bitTracker)
2531
+ elif (dtype == DType.UInt64):
2532
+ self.valueDecode = VDataUint64XorDecode(self.page, self.formula, self.bitTracker)
2533
+ elif (dtype == DType.Int64):
2534
+ self.valueDecode = VDataInt64XorDecode(self.page, self.formula, self.bitTracker)
2341
2535
  else:
2342
2536
  raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(dtype))
2343
2537
 
@@ -2431,7 +2625,7 @@ class VDataSeriesDecodeNamedArray(VDataSeriesDecode):
2431
2625
 
2432
2626
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2433
2627
  if (dtype != DType.String):
2434
- self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, True, self.formula, self.bitTracker)
2628
+ self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, False, self.formula, self.bitTracker)
2435
2629
  else:
2436
2630
  self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, True, self.formula, self.bitTracker)
2437
2631
 
@@ -3342,6 +3536,8 @@ class VDataFrame:
3342
3536
  Returns:
3343
3537
  array_like(array_like)
3344
3538
  '''
3539
+ if self._endTime - self._startTime > 3600000:
3540
+ return self.values(densifyRowsAhead, densifyOutputItv)
3345
3541
  oread = None
3346
3542
  if "all" == self._signalQueueMode:
3347
3543
  oread = VDataObjectsSeriesAllMode(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
@@ -3502,6 +3698,11 @@ class VDataDecoder:
3502
3698
  self._columnExpandMode = VDataUtils.getColumnExpandMode(self._columnExpandMode)
3503
3699
  self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(self._columnExpandMode)
3504
3700
 
3701
+ if self._queryStartTime ==0 and self._meta.queryStartTime() ==0 :
3702
+ self._queryStartTime = self._meta.storageStartTime()
3703
+ if self._queryEndTime ==0 and self._meta.queryEndTime() ==0 :
3704
+ self._queryEndTime = self._meta.storageEndTime()
3705
+
3505
3706
  formatVersion = self._meta.formatVersion()
3506
3707
  compressMethod = self._meta.compressMethod()
3507
3708
  # encryptMethod = self._meta.encryptMethod()
@@ -3513,7 +3714,10 @@ class VDataDecoder:
3513
3714
  logger.debug("file query start time: " + str(self._meta.queryStartTime()))
3514
3715
  logger.debug("file query end time: " + str(self._meta.queryEndTime()))
3515
3716
 
3516
- self.readKeys();
3717
+ if self._meta.getPageSize() != 0x8000:
3718
+ self.readKeys()
3719
+ else:
3720
+ self.readStfKeys()
3517
3721
 
3518
3722
  for name, key in self._keys.items():
3519
3723
  formulas[key._seriesid] = key._formula
@@ -3558,7 +3762,8 @@ class VDataDecoder:
3558
3762
  bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3559
3763
  length = _struct.unpack('<i', self._reader.read(4))[0] - 40 #include header
3560
3764
  self._reader.read(4) #ignore
3561
- else:
3765
+ elif (self._meta.getPageSize () != 0x8000):
3766
+ #default 64 page
3562
3767
  crc = _struct.unpack('<q', self._reader.read(8))[0]
3563
3768
  living = self._reader.read(1)[0] & 1
3564
3769
  self._reader.read(3) #ignore
@@ -3573,6 +3778,19 @@ class VDataDecoder:
3573
3778
  blobCrc = _struct.unpack('<q', self._reader.read(8))[0]
3574
3779
  blobCount = _struct.unpack('<i', self._reader.read(4))[0]
3575
3780
  blobDataLength = _struct.unpack('<i', self._reader.read(4))[0]
3781
+ else:
3782
+ # 32k page
3783
+ crc = _struct.unpack('<q', self._reader.read(8))[0]
3784
+ living = self._reader.read(1)[0] & 1
3785
+ self._reader.read(3) #ignore
3786
+ bucketCycle = _struct.unpack('<i', self._reader.read(4))[0]
3787
+ bucketStartTime = _struct.unpack('<q', self._reader.read(8))[0]
3788
+ bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3789
+ length = _struct.unpack('<i', self._reader.read(4))[0] - 48 #include header
3790
+
3791
+ position = _struct.unpack('<i', self._reader.read(4))[0]
3792
+ blobCount = _struct.unpack('<i', self._reader.read(4))[0]
3793
+ blobDataLength = _struct.unpack('<i', self._reader.read(4))[0]
3576
3794
 
3577
3795
  if (length <= 0):
3578
3796
  raise Exception('FORMAT_VDATA_BUCKET_LENGTH_INVALID')
@@ -3599,12 +3817,26 @@ class VDataDecoder:
3599
3817
  if (activePages <= 0):
3600
3818
  raise Exception('FORMAT_VDATA_ACTIVE_PAGES_ZERO')
3601
3819
 
3602
- for i in range(seriesCount):
3603
- seriesIds.append(_struct.unpack('<i', databytes[index:index+4])[0])
3604
- index += 4
3605
- for i in range(seriesCount):
3606
- storageIds.append(_struct.unpack('<q', databytes[index:index+8])[0])
3607
- index += 8
3820
+ if (self._meta.getPageSize() != 0x8000):
3821
+ for i in range(seriesCount):
3822
+ seriesIds.append(_struct.unpack('<i', databytes[index:index+4])[0])
3823
+ index += 4
3824
+ for i in range(seriesCount):
3825
+ storageIds.append(_struct.unpack('<q', databytes[index:index+8])[0])
3826
+ index += 8
3827
+ else:
3828
+ #5 byte storageId for 32k page size vsw
3829
+ for i in range(len(self._keyIds)):
3830
+ if (((databytes [index+(i>>3)] >> (i&0x7)) & 0x01) == 1):
3831
+ seriesIds.append(self._keyIds[i])
3832
+
3833
+ index = (len(self._keyIds) +7 )//8 + index
3834
+ for i in range(seriesCount):
3835
+ storageId = _struct.unpack('<i', databytes[index:index+4])[0]
3836
+ storageId = storageId& 0xffffffff
3837
+ storageId = ((databytes[index+4] &0xff) << 32 ) + storageId
3838
+ storageIds.append(storageId)
3839
+ index += 5
3608
3840
 
3609
3841
  blobSeriesIds =[]
3610
3842
  blobOffsetVec =[]
@@ -3626,7 +3858,7 @@ class VDataDecoder:
3626
3858
  raise Exception('FORMAT_VDATA_SERIES_NOT_FOUND' + str(seriesIds[i]))
3627
3859
  if (hasSignalFilter and not VDataSeriesUtils.serieMatchSignalFilter(self._signals, self._prefixes, formulas[seriesIds[i]], self._columnExpandMode, self._qualifiedExpandName)):
3628
3860
  continue
3629
- mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i )
3861
+ mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i , self._meta.getPageSize())
3630
3862
  if (mt._dataLength == 0 or mt._itemCount == 0):
3631
3863
  raise Exception('FORMAT_VDATA_SERIES_DATA_EMPTY')
3632
3864
 
@@ -3646,7 +3878,7 @@ class VDataDecoder:
3646
3878
 
3647
3879
  for i in range(len(seriesMetas)):
3648
3880
  mt = seriesMetas[i]
3649
- bstart = index + mt._pageIndex * VDataConstants.PAGE_LENGTH + mt._pageOffset
3881
+ bstart = index + mt._pageIndex * self._meta.getPageSize() + mt._pageOffset
3650
3882
  bend = bstart + mt._dataLength
3651
3883
  bt = databytes[bstart:bend]
3652
3884
 
@@ -3706,6 +3938,24 @@ class VDataDecoder:
3706
3938
  logger.warn("Replace series " + str(keyid) + " duplicate name {" + formula._name + "} with name {" + sname + "}")
3707
3939
  formula._name = sname
3708
3940
  self._keys[sname] = VDataSeriesSpec(sname, keyid, formula, keydesc)
3941
+
3942
+ def readStfKeys(self):
3943
+ (keysCount, length) = _struct.unpack('<ii', self._reader.read(8))
3944
+ if (keysCount <= 0):
3945
+ raise Exception('FORMAT_VDATA_KEYS_COUNT_INVALID')
3946
+ if (length <= 0):
3947
+ raise Exception('FORMAT_VDATA_KEYS_LENGTH_INVALID')
3948
+ databytes = self._reader.read(length)
3949
+ noneFormula = VDataFormulaNone('keys')
3950
+ decode = VDataInt32DeltaDecode(databytes, noneFormula, BitTracker())
3951
+
3952
+ self._keyIds = {}
3953
+ for i in range (keysCount):
3954
+ decode.next()
3955
+ self._keyIds[i] = decode.get()
3956
+ sname ="0x%08X"%self._keyIds[i]
3957
+ self._keys[sname] = VDataSeriesSpec(sname, self._keyIds[i] , VDataFormulaNone(sname), '')
3958
+
3709
3959
 
3710
3960
  def getUniqueName(self, keys, key):
3711
3961
  if key not in keys:
@@ -4094,7 +4344,7 @@ class VDataObjectsSeriesReader:
4094
4344
  time = decoder.time()
4095
4345
  millis = decoder.timestamp()
4096
4346
  pval = decoder.value()
4097
- if (millis >= self._startTime and millis <= self._endTime):
4347
+ if (millis >= decoder.qstime and millis <= decoder.qetime):
4098
4348
  rowidx = millis - self._startTime
4099
4349
  if (rawobjs[rowidx] is None):
4100
4350
  list = []
@@ -4137,7 +4387,7 @@ class VDataObjectsSeriesReader:
4137
4387
  time = decoder.time()
4138
4388
  millis = decoder.timestamp()
4139
4389
  pval = decoder.value()
4140
- if (millis >= self._startTime and millis <= self._endTime):
4390
+ if (millis >= decoder.qstime and millis <= decoder.qetime):
4141
4391
  rowidx = millis - self._startTime
4142
4392
  if (rawobjs[rowidx] is None):
4143
4393
  list = []
@@ -4198,12 +4448,11 @@ class VDataObjectsSeriesReader:
4198
4448
  rawobjs = [None] * rows # 2-d array of millisecond + data
4199
4449
  for i in range(rows):
4200
4450
  rawobjs[i] = [None] * self._colsize
4201
- rawobjs[i][0] = (startFreqBase + i) * frequency
4202
-
4203
- rowidx = 0
4204
- lastrowidx = -1
4451
+ rawobjs[i][0] = (startFreqBase + i) * frequency
4205
4452
 
4206
4453
  for pname in self._series:
4454
+ rowidx = 0
4455
+ lastrowidx = -1
4207
4456
  serie = self._series[pname]
4208
4457
  colidx = self._indices.get(serie.name())
4209
4458
  decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
@@ -4243,10 +4492,10 @@ class VDataObjectsSeriesReader:
4243
4492
  rawobjs[i] = [None] * self._colsize
4244
4493
  rawobjs[i][0] = (startFreqBase + i) * frequency
4245
4494
 
4246
- rowidx = 0;
4247
- lastrowidx = -1;
4495
+ rowidx = 0
4248
4496
 
4249
4497
  for pname in self._series:
4498
+ lastrowidx = -1
4250
4499
  serie = self._series[pname]
4251
4500
  colidx = self._indices.get(serie.name())
4252
4501
  decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
@@ -4764,6 +5013,7 @@ class VDataReader:
4764
5013
  qstime = 0 # query start time
4765
5014
  qetime = 0 # query end time
4766
5015
  extlength = 0
5016
+ pageSizeCode = 0
4767
5017
 
4768
5018
  if (formatVersion > 0):
4769
5019
  if (formatVersion < 26):
@@ -4771,10 +5021,14 @@ class VDataReader:
4771
5021
  else:
4772
5022
  (sstime, setime, qstime, qetime) = _struct.unpack('<qqqq', _reader.read(32))
4773
5023
  extlength = _struct.unpack('<H', _reader.read(2))[0]
4774
- _reader.read(6) # reserved
5024
+ if (formatVersion > 26):
5025
+ pageSizeCode = _reader.read(1)[0]
5026
+ _reader.read(5) # reserved
5027
+ else:
5028
+ _reader.read(6) # reserved
4775
5029
 
4776
5030
  extbytes = _reader.read(extlength) if (extlength > 0) else []
4777
- return VDataMeta(formatVersion=formatVersion, compressMethod=compressMethod, encryptMethod=encryptMethod, blocksCount=blocksCount, storageStartTime=sstime, storageEndTime=setime, queryStartTime=qstime, queryEndTime=qetime, extendedInfo=extbytes)
5031
+ return VDataMeta(formatVersion=formatVersion, compressMethod=compressMethod, encryptMethod=encryptMethod, blocksCount=blocksCount, storageStartTime=sstime, storageEndTime=setime, queryStartTime=qstime, queryEndTime=qetime, extendedInfo=extbytes, pageSizeCode = pageSizeCode)
4778
5032
 
4779
5033
 
4780
5034
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.9.3.2
3
+ Version: 2.9.6.1
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -1,3 +1,4 @@
1
+ LICENSE
1
2
  README.rst
2
3
  setup.py
3
4
  exceeddata/sdk/__init__.py
@@ -33,7 +33,7 @@
33
33
  from setuptools import setup
34
34
 
35
35
  setup(name='exceeddata-sdk-vdata',
36
- version='2.9.3.2',
36
+ version='2.9.6.1',
37
37
  description='ExceedData vData SDK for Python',
38
38
  url='http://www.smartsct.com',
39
39
  author='Nick Xie',